[drill] branch master updated: DRILL-7317: Close ClassLoaders used for udf jars uploading when closing FunctionImplementationRegistry
This is an automated email from the ASF dual-hosted git repository.
volodymyr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 77cf7e2 DRILL-7317: Close ClassLoaders used for udf jars uploading
when closing FunctionImplementationRegistry
77cf7e2 is described below
commit 77cf7e2ee61fb40e7efd85148ac76947d13dda38
Author: Volodymyr Vysotskyi
AuthorDate: Fri Jul 5 16:13:54 2019 +0300
DRILL-7317: Close ClassLoaders used for udf jars uploading when closing
FunctionImplementationRegistry
- Fix issue with caching DrillMergeProjectRule and
FunctionImplementationRegistry when different drillbits are started within the
same JVM
---
.../expr/fn/FunctionImplementationRegistry.java| 3 +-
.../expr/fn/registry/FunctionRegistryHolder.java | 75 ++
.../expr/fn/registry/LocalFunctionRegistry.java| 7 +-
.../planner/logical/DrillMergeProjectRule.java | 6 +-
4 files changed, 70 insertions(+), 21 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
index 30c194a..4210067 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
@@ -455,7 +455,7 @@ public class FunctionImplementationRegistry implements
FunctionLookupContext, Au
return RunTimeScan.dynamicPackageScan(drillConfig,
Sets.newHashSet(urls));
} finally {
if (markerFileConnection instanceof JarURLConnection) {
-((JarURLConnection)
markerFile.openConnection()).getJarFile().close();
+((JarURLConnection) markerFileConnection).getJarFile().close();
}
}
}
@@ -592,6 +592,7 @@ public class FunctionImplementationRegistry implements
FunctionLookupContext, Au
*/
@Override
public void close() {
+localFunctionRegistry.close();
if (deleteTmpDir) {
FileUtils.deleteQuietly(tmpDir);
} else {
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolder.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolder.java
index d0383e9..f378d45 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolder.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolder.java
@@ -29,6 +29,8 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -83,7 +85,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
* function holder for upper(VARCHAR-REQUIRED) is {@link
DrillFuncHolder} initiated for each function.
*
*/
-public class FunctionRegistryHolder {
+public class FunctionRegistryHolder implements AutoCloseable {
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(FunctionRegistryHolder.class);
@@ -376,23 +378,13 @@ public class FunctionRegistryHolder {
return;
}
+boolean isClosed = false;
for (Map.Entry> functionEntry : jar.entrySet()) {
final String function = functionEntry.getKey();
Map functionHolders = functions.get(function);
Queue functionSignatures = functionEntry.getValue();
- for (Map.Entry entry :
functionHolders.entrySet()) {
-if (functionSignatures.contains(entry.getKey())) {
- ClassLoader classLoader = entry.getValue().getClassLoader();
- if (classLoader instanceof AutoCloseable) {
-try {
- ((AutoCloseable) classLoader).close();
-} catch (Exception e) {
- logger.warn("Problem during closing class loader", e);
-}
- }
- break;
-}
- }
+ // closes class loader only one time
+ isClosed = !isClosed && closeClassLoader(function, functionSignatures);
functionHolders.keySet().removeAll(functionSignatures);
if (functionHolders.isEmpty()) {
@@ -400,4 +392,59 @@ public class FunctionRegistryHolder {
}
}
}
+
+ @Override
+ public void close() {
+try (@SuppressWarnings("unused") Closeable lock = writeLock.open()) {
+ jars.forEach((jarName, jar) -> {
+if (!LocalFunctionRegistry.BUILT_IN.equals(jarName)) {
+ for (Map.Entry> functionEntry :
jar.entrySet()) {
+if (closeClassLoader(functionEntry.getKey(),
functionEntry.getValue())) {
+ // class loader is closed, iterates to another
[drill] branch master updated: DRILL-7306: Disable schema-only batch for new scan framework
This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git The following commit(s) were added to refs/heads/master by this push: new 66e8dd9 DRILL-7306: Disable schema-only batch for new scan framework 66e8dd9 is described below commit 66e8dd9f05ea068e27b272ed1b787004213c29e6 Author: Paul Rogers AuthorDate: Sun Jun 23 22:10:54 2019 -0700 DRILL-7306: Disable schema-only batch for new scan framework The EVF framework is set up to return a "fast schema" empty batch with only schema as its first batch because, when the code was written, it seemed that's how we wanted operators to work. However, DRILL-7305 notes that many operators cannot handle empty batches. Since the empty-batch bugs show that Drill does not, in fact, provide a "fast schema" batch, this ticket asks to disable the feature in the new scan framework. The feature is disabled with a config option; it can be re-enabled if ever it is needed. SQL differentiates between two subtle cases, and both are supported by this change. 1. Empty results: the query found a schema, but no rows are returned. If no reader returns any rows, but at least one reader provides a schema, then the scan returns an empty batch with the schema. 2. Null results: the query found no schema or rows. No schema is returned. If no reader returns rows or schema, then the scan returns no batch: it instead immediately returns a DONE status. For CSV, an empty file with headers returns the null result set (because we don't know the schema.) An empty CSV file without headers returns an empty result set because we do know the schema: it will always be the columns array. Old tests validate the original schema-batch mode, new tests added to validate the no-schema-batch mode. --- .../exec/physical/impl/filter/FilterTemplate2.java | 6 +- .../physical/impl/protocol/OperatorDriver.java | 57 +-- .../impl/protocol/OperatorRecordBatch.java | 5 +- .../exec/physical/impl/protocol/SchemaTracker.java | 6 +- .../impl/protocol/VectorContainerAccessor.java | 30 -- .../drill/exec/physical/impl/scan/ReaderState.java | 64 .../exec/physical/impl/scan/ScanOperatorExec.java | 109 +++-- .../impl/scan/columns/ColumnsScanFramework.java| 3 +- .../physical/impl/scan/file/FileScanFramework.java | 4 +- .../impl/scan/framework/ManagedScanFramework.java | 5 + .../impl/scan/project/ScanSchemaOrchestrator.java | 64 +++- .../physical/impl/scan/project/SchemaSmoother.java | 3 +- .../apache/drill/exec/record/VectorContainer.java | 36 --- .../exec/store/dfs/easy/EasyFormatPlugin.java | 76 +++--- .../exec/store/easy/text/TextFormatPlugin.java | 9 +- .../easy/text/reader/CompliantTextBatchReader.java | 2 +- .../apache/drill/TestSchemaWithTableFunction.java | 11 +-- .../org/apache/drill/exec/TestEmptyInputSql.java | 6 +- .../impl/protocol/TestOperatorRecordBatch.java | 24 ++--- .../impl/scan/BaseScanOperatorExecTest.java| 30 +++--- .../exec/physical/impl/scan/ScanTestUtils.java | 27 +++-- .../exec/physical/impl/scan/TestColumnsArray.java | 6 +- .../impl/scan/TestColumnsArrayFramework.java | 6 +- .../physical/impl/scan/TestFileScanFramework.java | 6 +- .../physical/impl/scan/TestScanOperExecBasics.java | 11 ++- .../impl/scan/TestScanOperExecEarlySchema.java | 52 +- .../impl/scan/TestScanOperExecLateSchema.java | 44 + .../impl/scan/TestScanOperExecSmoothing.java | 21 .../impl/scan/TestScanOrchestratorEarlySchema.java | 27 ++--- .../impl/scan/TestScanOrchestratorLateSchema.java | 5 +- .../impl/scan/TestScanOrchestratorMetadata.java| 11 ++- .../impl/scan/project/TestSchemaSmoothing.java | 3 +- .../store/easy/text/compliant/BaseCsvTest.java | 11 +++ .../store/easy/text/compliant/TestCsvHeader.java | 10 +- .../easy/text/compliant/TestCsvWithHeaders.java| 66 - .../easy/text/compliant/TestCsvWithSchema.java | 2 +- .../easy/text/compliant/TestCsvWithoutHeaders.java | 43 ++-- .../easy/text/compliant/TestPartitionRace.java | 35 --- .../apache/drill/exec/store/log/TestLogReader.java | 4 - .../java/org/apache/drill/test/QueryBuilder.java | 35 +-- .../apache/drill/test/rowSet/RowSetBuilder.java| 10 +- .../apache/drill/test/rowSet/RowSetUtilities.java | 2 +- .../drill/exec/record/MaterializedField.java | 4 +- .../drill/exec/record/metadata/ColumnBuilder.java | 17 +++- .../record/metadata/PrimitiveColumnMetadata.java | 18 +++- 45 files changed, 764 insertions(+), 262 deletions(-) diff --git
