[drill] 05/05: DRILL-7236: SqlLine 1.8 upgrade
This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 2766e653cda8b1de817b234c66b0058e707750d0 Author: Arina Ielchiieva AuthorDate: Mon Jun 3 17:15:09 2019 +0300 DRILL-7236: SqlLine 1.8 upgrade closes #1804 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a1ec48e..e141ea0 100644 --- a/pom.xml +++ b/pom.xml @@ -53,7 +53,7 @@ 1.18.0-drill-r2 1.13.0 3.0.11 -1.7.0 +1.8.0 2.9.5 2.9.5 3.4.12
[drill] branch master updated (2615d68 -> 2766e65)
This is an automated email from the ASF dual-hosted git repository.
arina pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 2615d68 DRILL-4782 / DRILL-7139: Fix DATE_ADD and TO_TIME functions
new 8a7007f DRILL-7258: Remove field width limit for text reader
new 1bf7f15 DRILL-7261: Simplify Easy framework config for new scan
new 20ae96a DRILL-7279: Enable provided schema for text files without
headers
new 0dbd501 DRILL-7278: Refactor result set loader projection mechanism
new 2766e65 DRILL-7236: SqlLine 1.8 upgrade
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../physical/impl/scan/file/FileScanFramework.java | 8 +-
.../impl/scan/framework/ManagedScanFramework.java | 7 +
.../impl/scan/framework/SchemaNegotiator.java | 7 +
.../impl/scan/framework/SchemaNegotiatorImpl.java | 5 +
.../scan/project/AbstractUnresolvedColumn.java | 38 +-
.../scan/project/ReaderSchemaOrchestrator.java | 18 +-
.../impl/scan/project/ResolvedNullColumn.java | 12 +
.../impl/scan/project/ScanLevelProjection.java | 65 ++-
.../impl/scan/project/ScanSchemaOrchestrator.java | 68 +--
.../scan/project/WildcardSchemaProjection.java | 35 +-
.../physical/impl/scan/project/package-info.java | 182 +-
.../project/projSet/AbstractProjectionSet.java | 80 +++
.../scan/project/projSet/AbstractReadColProj.java | 50 ++
.../scan/project/projSet/EmptyProjectionSet.java | 24 +-
.../project/projSet/ExplicitProjectionSet.java | 109
.../projSet/ProjectedMapColumn.java} | 21 +-
.../scan/project/projSet/ProjectedReadColumn.java | 78 +++
.../scan/project/projSet/ProjectionSetBuilder.java | 101
.../scan/project/projSet/ProjectionSetFactory.java | 79 +++
.../impl/scan/project/projSet/TypeConverter.java | 173 ++
.../project/projSet/UnprojectedReadColumn.java}| 23 +-
.../project/projSet/WildcardProjectionSet.java | 55 ++
.../impl/scan/project/projSet/package-info.java| 99
.../drill/exec/physical/rowSet/ProjectionSet.java | 105
.../exec/physical/rowSet/impl/ColumnBuilder.java | 218 ++-
.../exec/physical/rowSet/impl/ColumnState.java | 3 +-
.../exec/physical/rowSet/impl/ContainerState.java | 18 +-
.../rowSet/impl/DefaultSchemaTransformer.java | 77 ---
.../drill/exec/physical/rowSet/impl/ListState.java | 7 +-
.../exec/physical/rowSet/impl/LoaderInternals.java | 8 +
.../exec/physical/rowSet/impl/OptionBuilder.java | 47 +-
.../physical/rowSet/impl/RepeatedListState.java| 5 +-
.../physical/rowSet/impl/ResultSetLoaderImpl.java | 36 +-
.../physical/rowSet/impl/SchemaTransformer.java| 46 --
.../rowSet/impl/SchemaTransformerImpl.java | 194 ---
.../exec/physical/rowSet/impl/TupleState.java | 12 +-
.../exec/physical/rowSet/impl/UnionState.java | 5 +-
.../exec/physical/rowSet/impl/WriterIndexImpl.java | 3 +
.../rowSet/project/ImpliedTupleRequest.java| 8 +-
.../physical/rowSet/project/ProjectionType.java| 178 ++
.../rowSet/project/RequestedColumnImpl.java| 29 +-
.../physical/rowSet/project/RequestedTuple.java| 14 +-
.../rowSet/project/RequestedTupleImpl.java | 30 +-
.../record/metadata/AbstractColumnMetadata.java| 34 +-
.../exec/store/dfs/easy/EasyFormatPlugin.java | 412 +++---
.../exec/store/easy/text/TextFormatPlugin.java | 193 ---
.../easy/text/compliant/v3/BaseFieldOutput.java| 52 +-
.../compliant/v3/CompliantTextBatchReader.java | 17 +-
.../easy/text/compliant/v3/FieldVarCharOutput.java | 14 +-
.../text/compliant/v3/RepeatedVarCharOutput.java | 9 +-
.../store/easy/text/compliant/v3/TextInput.java| 11 +-
.../text/compliant/v3/TextParsingSettingsV3.java | 282 --
.../store/easy/text/compliant/v3/TextReader.java | 43 +-
.../physical/impl/scan/TestFileScanFramework.java | 1 -
.../impl/scan/TestScanOperExecOuputSchema.java | 75 ++-
.../impl/scan/TestScanOrchestratorEarlySchema.java | 8 +-
.../impl/scan/project/TestScanLevelProjection.java | 208 +--
.../scan/project/projSet/TestProjectionSet.java| 625 +
.../impl/TestResultSetLoaderEmptyProject.java | 13 +-
.../rowSet/impl/TestResultSetLoaderOverflow.java | 76 ++-
.../rowSet/impl/TestResultSetLoaderProjection.java | 92 +--
.../impl/TestResultSetLoaderTypeConversion.java| 22 +-
.../{impl => project}/TestProjectedTuple.java | 65 ++-
.../rowSet/project/TestProjectionType.java | 154 +
.../record/metadata/TestMetadataProperties.java| 20 -
.../store/easy/text/compliant/BaseCsvTest.java | 43 ++
[drill] 03/05: DRILL-7279: Enable provided schema for text files without headers
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 20ae96a4d0de23d329de1b5534683fdaac8816a7
Author: Paul Rogers
AuthorDate: Sun May 26 23:32:40 2019 -0700
DRILL-7279: Enable provided schema for text files without headers
* Allows a provided schema for text files without headers. The
provided schema columns replace the `columns` column that is
normally used.
* Allows customizing text format properties using table properties.
The table properties "override" properties set in the plugin config.
* Added unit tests for the newly supported use cases.
* Fixed bug in quote escape handling.
closes #1798
---
.../impl/scan/project/ScanLevelProjection.java | 2 +-
.../impl/scan/project/ScanSchemaOrchestrator.java | 2 +-
.../exec/store/dfs/easy/EasyFormatPlugin.java | 140 ++-
.../exec/store/easy/text/TextFormatPlugin.java | 117 +-
.../compliant/v3/CompliantTextBatchReader.java | 17 +-
.../store/easy/text/compliant/v3/TextInput.java| 2 -
.../text/compliant/v3/TextParsingSettingsV3.java | 282 ++---
.../store/easy/text/compliant/v3/TextReader.java | 43 +-
.../store/easy/text/compliant/BaseCsvTest.java | 25 ++
.../easy/text/compliant/TestCsvIgnoreHeaders.java | 11 +-
.../text/compliant/TestCsvTableProperties.java | 451 +
.../easy/text/compliant/TestCsvWithSchema.java | 27 +-
.../easy/text/compliant/TestCsvWithoutHeaders.java | 10 +-
.../exec/record/metadata/AbstractPropertied.java | 13 +-
.../drill/exec/record/metadata/Propertied.java | 3 +-
15 files changed, 917 insertions(+), 228 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanLevelProjection.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanLevelProjection.java
index 7718119..4a02b33 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanLevelProjection.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanLevelProjection.java
@@ -456,7 +456,7 @@ public class ScanLevelProjection {
if (hasOutputSchema()) {
projectionType =
- outputSchema.getBooleanProperty(TupleMetadata.IS_STRICT_SCHEMA_PROP)
+ outputSchema.booleanProperty(TupleMetadata.IS_STRICT_SCHEMA_PROP)
? ScanProjectionType.STRICT_SCHEMA_WILDCARD
: ScanProjectionType.SCHEMA_WILDCARD;
} else if (wildcardPosn != -1) {
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanSchemaOrchestrator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanSchemaOrchestrator.java
index a315a3f..37f7c75 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanSchemaOrchestrator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanSchemaOrchestrator.java
@@ -322,7 +322,7 @@ public class ScanSchemaOrchestrator {
// Use only implicit conversions
schemaTransformer = new SchemaTransformerImpl(
builder.outputSchema, builder.conversionProps);
-if
(builder.outputSchema.getBooleanProperty(TupleMetadata.IS_STRICT_SCHEMA_PROP)) {
+if
(builder.outputSchema.booleanProperty(TupleMetadata.IS_STRICT_SCHEMA_PROP)) {
allowRequiredNulls = true;
}
} else {
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
index a170122..3c79aea 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
@@ -147,6 +147,142 @@ public abstract class EasyFormatPlugin implements
.addContext(errorContext())
.build(logger);
}
+
+ return new ScanBatch(context, oContext, readers, implicitColumns);
+}
+
+/**
+ * Create a record reader given a file system, a file description and other
+ * information. For backward compatibility, calls the plugin method by
+ * default.
+ *
+ * @param plugin
+ * the plugin creating the scan
+ * @param context
+ * fragment context for the fragment running the scan
+ * @param dfs
+ * Drill's distributed file system facade
+ * @param fileWork
+ * description of the file to scan
+ * @param columns
+ * list of columns to project
+ * @param userName
+ * the name of the user performing the scan
+ * @return a scan operator
+ * @throws ExecutionSetupException
+
[drill] 01/05: DRILL-7258: Remove field width limit for text reader
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 8a7007f03397849b555a74297a6a637293958cc5
Author: Paul Rogers
AuthorDate: Thu May 30 18:43:09 2019 -0700
DRILL-7258: Remove field width limit for text reader
The V2 text reader enforced a limit of 64K characters when using
column headers, but not when using the columns[] array. The V3 reader
enforced the 64K limit in both cases.
This patch removes the limit in both cases. The limit now is the
16MB vector size limit. With headers, no one column can exceed 16MB.
With the columns[] array, no one row can exceed 16MB. (The 16MB
limit is set by the Netty memory allocator.)
Added an "appendBytes()" method to the scalar column writer which adds
additional bytes to those already written for a specific column or
array element value. The method is implemented for VarChar, Var16Char
and VarBinary vectors. It throws an exception for all other types.
When used with a type conversion shim, the appendBytes() method throws
an exception. This should be OK because, the previous setBytes() should
have failed because a huge value is not acceptable for numeric or date
types conversions.
Added unit tests of the append feature, and for the append feature in
the batch overflow case (when appending bytes causes the vector or
batch to overflow.) Also added tests to verify the lack of column width
limit with the text reader, both with and without headers.
closes #1802
---
.../exec/physical/rowSet/impl/WriterIndexImpl.java | 3 +
.../easy/text/compliant/v3/BaseFieldOutput.java| 52 ++---
.../easy/text/compliant/v3/FieldVarCharOutput.java | 12 +--
.../text/compliant/v3/RepeatedVarCharOutput.java | 7 +-
.../store/easy/text/compliant/v3/TextInput.java| 9 +--
.../rowSet/impl/TestResultSetLoaderOverflow.java | 76 +-
.../store/easy/text/compliant/BaseCsvTest.java | 20 +
.../easy/text/compliant/TestCsvWithHeaders.java| 25 ++
.../easy/text/compliant/TestCsvWithoutHeaders.java | 29 +++
.../apache/drill/test/rowSet/RowSetWriterImpl.java | 3 +
.../drill/test/rowSet/test/PerformanceTool.java| 5 +-
.../test/rowSet/test/TestFixedWidthWriter.java | 3 +
.../test/rowSet/test/TestScalarAccessors.java | 89 ++
.../main/codegen/templates/ColumnAccessors.java| 11 +++
.../exec/vector/accessor/ColumnWriterIndex.java| 13 +++-
.../drill/exec/vector/accessor/ScalarWriter.java | 1 +
.../accessor/convert/AbstractWriteConverter.java | 5 ++
.../accessor/writer/AbstractArrayWriter.java | 7 +-
.../vector/accessor/writer/BaseScalarWriter.java | 5 ++
.../vector/accessor/writer/BaseVarWidthWriter.java | 7 ++
.../exec/vector/accessor/writer/MapWriter.java | 1 +
.../accessor/writer/NullableScalarWriter.java | 8 ++
.../accessor/writer/OffsetVectorWriterImpl.java| 6 ++
.../vector/accessor/writer/ScalarArrayWriter.java | 3 +
.../accessor/writer/dummy/DummyScalarWriter.java | 3 +
25 files changed, 376 insertions(+), 27 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/WriterIndexImpl.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/WriterIndexImpl.java
index 9fb3e4e..6119791 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/WriterIndexImpl.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/WriterIndexImpl.java
@@ -106,6 +106,9 @@ class WriterIndexImpl implements ColumnWriterIndex {
public void nextElement() { }
@Override
+ public void prevElement() { }
+
+ @Override
public ColumnWriterIndex outerIndex() { return null; }
@Override
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/v3/BaseFieldOutput.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/v3/BaseFieldOutput.java
index 6bf0bb6..5dd4284 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/v3/BaseFieldOutput.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/v3/BaseFieldOutput.java
@@ -17,13 +17,17 @@
*/
package org.apache.drill.exec.store.easy.text.compliant.v3;
-import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.physical.rowSet.RowSetLoader;
+import org.apache.drill.exec.vector.accessor.ScalarWriter;
public abstract class BaseFieldOutput extends TextOutput {
- private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(BaseFieldOutput.class);
- private static final int MAX_FIELD_LENGTH = 1024 * 64;
+ /**
+ * Width of the per-field data buffer. Fields can be larger.
+ * In that
[drill] 02/05: DRILL-7261: Simplify Easy framework config for new scan
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 1bf7f1525374c9e414f89d5a17056e860a125510
Author: Paul Rogers
AuthorDate: Sat May 25 20:40:22 2019 -0700
DRILL-7261: Simplify Easy framework config for new scan
Most format plugins are created using the Easy format plugin. A recent
change added support for the "row set" scan framework. After converting
the text and log reader plugins, it became clear that the setup code
could be made simpler.
* Add the user name to the "file scan" framework.
* Pass the file system, split and user name to the batch reader via
the "schema negotiator" rather than via the constructor.
* Create the traditional "scan batch" scan or the new row-set scan via
functions instead of classes.
* Add Easy config option and method to choose the kind of scan
framework.
* Add Easy config options for some newer options such as whether the
plugin supports statistics.
Simplified reader creation
* The batch reader can be created just by overriding a method.
* A default error context is provided if the plugin does not provide
one.
Tested by running all unit tests for the CSV reader which is based on
the new framework, and by testing the converted log reader (that reader
is not part of this commit.)
closes #1796
---
.../physical/impl/scan/file/FileScanFramework.java | 8 +-
.../impl/scan/framework/ManagedScanFramework.java | 7 +
.../impl/scan/framework/SchemaNegotiator.java | 7 +
.../impl/scan/framework/SchemaNegotiatorImpl.java | 5 +
.../exec/store/dfs/easy/EasyFormatPlugin.java | 408 ++---
.../exec/store/easy/text/TextFormatPlugin.java | 142 +++
.../easy/text/compliant/TestCsvWithHeaders.java| 4 +-
7 files changed, 282 insertions(+), 299 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/file/FileScanFramework.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/file/FileScanFramework.java
index f69aa40..761f68b 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/file/FileScanFramework.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/file/FileScanFramework.java
@@ -62,7 +62,8 @@ import org.apache.hadoop.mapred.FileSplit;
public class FileScanFramework extends ManagedScanFramework {
- private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(FileScanFramework.class);
+ private static final org.slf4j.Logger logger =
+ org.slf4j.LoggerFactory.getLogger(FileScanFramework.class);
/**
* The file schema negotiator adds no behavior at present, but is
@@ -80,7 +81,6 @@ public class FileScanFramework extends ManagedScanFramework {
/**
* Gives the Drill file system for this operator.
*/
-
DrillFileSystem fileSystem();
/**
@@ -186,6 +186,10 @@ public class FileScanFramework extends
ManagedScanFramework {
return newReader();
}
+public CustomErrorContext errorContext() {
+ return fileFramework == null ? null : fileFramework.errorContext();
+}
+
public abstract ManagedReader newReader();
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/ManagedScanFramework.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/ManagedScanFramework.java
index 52203da..a7a46fe 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/ManagedScanFramework.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/ManagedScanFramework.java
@@ -135,10 +135,17 @@ public class ManagedScanFramework implements
ScanOperatorEvents {
public static class ScanFrameworkBuilder extends ScanOrchestratorBuilder {
protected ReaderFactory readerFactory;
+protected String userName;
public void setReaderFactory(ReaderFactory readerFactory) {
this.readerFactory = readerFactory;
}
+
+public ReaderFactory readerFactory() { return readerFactory; }
+
+public void setUserName(String userName) {
+ this.userName = userName;
+}
}
// Inputs
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiator.java
index bc303ec..74ea512 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiator.java
@@ -64,8 +64,15 @@ public interface SchemaNegotiator {
* Specify an advanced error context which
