DRILL-1166: Session option to enable/disable debug information in runtime generated Java code
+ By default, debug options are enabled but can be disabled by setting the session option `exec.java_compiler_debug` to false. + Allow the defaults for compiler options to be set through configuration. Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/97a9a4c1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/97a9a4c1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/97a9a4c1 Branch: refs/heads/master Commit: 97a9a4c155da4f5a3604c24db7e8ab5ec47797ce Parents: 2a4d6f1 Author: Aditya Kishore <[email protected]> Authored: Mon Jul 21 16:09:44 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Fri Jul 25 14:34:18 2014 -0700 ---------------------------------------------------------------------- .../exec/compile/AbstractClassCompiler.java | 29 +++----- .../drill/exec/compile/ClassCompiler.java | 30 -------- .../drill/exec/compile/JDKClassCompiler.java | 20 ++---- .../drill/exec/compile/JaninoClassCompiler.java | 5 +- .../drill/exec/compile/QueryClassLoader.java | 33 ++++++--- .../apache/drill/exec/ops/FragmentContext.java | 2 +- .../server/options/SystemOptionManager.java | 1 + .../src/main/resources/drill-module.conf | 5 ++ .../exec/compile/TestClassTransformation.java | 76 +++++++++++++++----- .../exec/compile/TestLargeFileCompilation.java | 8 ++- .../exec/fn/impl/TestNewMathFunctions.java | 1 + .../physical/impl/TestComparisonFunctions.java | 1 + .../impl/TestImplicitCastFunctions.java | 1 + .../exec/physical/impl/TestSimpleFunctions.java | 3 + .../exec/physical/impl/TestStringFunctions.java | 1 + .../physical/impl/common/TestHashTable.java | 1 + .../physical/impl/filter/TestSimpleFilter.java | 2 + .../exec/physical/impl/join/TestMergeJoin.java | 1 + .../physical/impl/limit/TestSimpleLimit.java | 3 + .../impl/project/TestSimpleProjection.java | 1 + .../exec/physical/impl/sort/TestSimpleSort.java | 2 + .../physical/impl/svremover/TestSVRemover.java | 1 + .../physical/impl/union/TestSimpleUnion.java | 1 + .../drill/jdbc/test/JdbcTestQueryBase.java | 2 +- 24 files changed, 132 insertions(+), 98 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/main/java/org/apache/drill/exec/compile/AbstractClassCompiler.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/AbstractClassCompiler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/AbstractClassCompiler.java index 91588b8..98c51c6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/AbstractClassCompiler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/AbstractClassCompiler.java @@ -24,17 +24,13 @@ import org.apache.drill.exec.compile.ClassTransformer.ClassNames; import org.apache.drill.exec.exception.ClassTransformationException; import org.codehaus.commons.compiler.CompileException; -public abstract class AbstractClassCompiler implements ClassCompiler { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractClassCompiler.class); +public abstract class AbstractClassCompiler { + protected boolean debug = false; - protected boolean debugLines = true; - protected boolean debugVars = true; - protected boolean debugSource = true; - - protected abstract byte[][] getByteCode(ClassNames className, String sourcecode) - throws CompileException, IOException, ClassNotFoundException, ClassTransformationException; + AbstractClassCompiler(boolean debug) { + this.debug = debug; + } - @Override public byte[][] getClassByteCode(ClassNames className, String sourceCode) throws CompileException, IOException, ClassNotFoundException, ClassTransformationException { if(getLogger().isDebugEnabled()){ @@ -43,15 +39,9 @@ public abstract class AbstractClassCompiler implements ClassCompiler { return getByteCode(className, sourceCode); } - public void setDebuggingOptions(boolean debugSource, boolean debugLines, boolean debugVars) { - this.debugSource = debugSource; - this.debugLines = debugLines; - this.debugVars = debugVars; - updateDebugOptions(); - } - protected String prefixLineNumbers(String code) { - if (!debugLines) return code; + if (!debug) return code; + StringBuilder out = new StringBuilder(); int i = 1; for (String line : code.split("\n")) { @@ -68,8 +58,9 @@ public abstract class AbstractClassCompiler implements ClassCompiler { return out.toString(); } - protected void updateDebugOptions() { } // default no-op implementation + protected abstract byte[][] getByteCode(ClassNames className, String sourcecode) + throws CompileException, IOException, ClassNotFoundException, ClassTransformationException; - protected org.slf4j.Logger getLogger() { return logger; } + protected abstract org.slf4j.Logger getLogger(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompiler.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompiler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompiler.java deleted file mode 100644 index fb080f8..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompiler.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.compile; - -import java.io.IOException; - -import org.apache.drill.exec.compile.ClassTransformer.ClassNames; -import org.apache.drill.exec.exception.ClassTransformationException; -import org.codehaus.commons.compiler.CompileException; - -interface ClassCompiler { - - public abstract byte[][] getClassByteCode(ClassNames className, String sourcecode) throws CompileException, IOException, ClassNotFoundException, ClassTransformationException ; - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JDKClassCompiler.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JDKClassCompiler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JDKClassCompiler.java index 2bbf4f8..e1f52ca 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JDKClassCompiler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JDKClassCompiler.java @@ -36,19 +36,20 @@ import com.google.common.collect.Lists; class JDKClassCompiler extends AbstractClassCompiler { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JDKClassCompiler.class); - private Collection<String> compilerOptions; - private DiagnosticListener<JavaFileObject> listener; + private final Collection<String> compilerOptions; + private final DiagnosticListener<JavaFileObject> listener; private final JavaCompiler compiler; private final DrillJavaFileManager fileManager; - public JDKClassCompiler(ClassLoader classLoader) { + public JDKClassCompiler(ClassLoader classLoader, boolean debug) { + super(debug); this.compiler = ToolProvider.getSystemJavaCompiler(); if (compiler == null) { throw new UnsupportedOperationException("JDK Java compiler not available - probably you're running a JRE, not a JDK"); } this.listener = new DrillDiagnosticListener(); this.fileManager = new DrillJavaFileManager(compiler.getStandardFileManager(listener, null, Charsets.UTF_8), classLoader); - updateDebugOptions(); + this.compilerOptions = Lists.newArrayList(this.debug ? "-g:source,lines,vars" : "-g:none"); } @Override @@ -80,17 +81,6 @@ class JDKClassCompiler extends AbstractClassCompiler { } } - protected void updateDebugOptions() { - StringBuilder sb = new StringBuilder("-g:"); - if (this.debugSource) sb.append("source,"); - if (this.debugLines) sb.append("lines,"); - if (this.debugVars) sb.append("vars,"); - if (sb.length() == 3) { // "-g:" - sb.append("none,"); - } - compilerOptions = Lists.newArrayList(sb.substring(0, sb.length()-1)); - } - protected org.slf4j.Logger getLogger() { return logger; } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java index 4b17598..aa16b83 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java @@ -36,7 +36,8 @@ public class JaninoClassCompiler extends AbstractClassCompiler { private IClassLoader compilationClassLoader; - public JaninoClassCompiler(ClassLoader parentClassLoader) { + public JaninoClassCompiler(ClassLoader parentClassLoader, boolean debug) { + super(debug); this.compilationClassLoader = new ClassLoaderIClassLoader(parentClassLoader); } @@ -46,7 +47,7 @@ public class JaninoClassCompiler extends AbstractClassCompiler { Scanner scanner = new Scanner((String) null, reader); Java.CompilationUnit compilationUnit = new Parser(scanner).parseCompilationUnit(); ClassFile[] classFiles = new UnitCompiler(compilationUnit, compilationClassLoader) - .compileUnit(this.debugSource, this.debugLines, this.debugVars); + .compileUnit(this.debug, this.debug, this.debug); byte[][] byteCodes = new byte[classFiles.length][]; for(int i = 0; i < classFiles.length; i++){ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java index 8f1abcb..8de14ec 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java @@ -24,12 +24,14 @@ import java.util.Arrays; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; +import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.ExpressionParsingException; import org.apache.drill.exec.compile.ClassTransformer.ClassNames; import org.apache.drill.exec.exception.ClassTransformationException; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.OptionValidator; import org.apache.drill.exec.server.options.OptionValue; +import org.apache.drill.exec.server.options.TypeValidators.BooleanValidator; import org.apache.drill.exec.server.options.TypeValidators.LongValidator; import org.apache.drill.exec.server.options.TypeValidators.StringValidator; import org.codehaus.commons.compiler.CompileException; @@ -55,8 +57,15 @@ public class QueryClassLoader extends URLClassLoader { } }; + public static final String JAVA_COMPILER_DEBUG_OPTION = "exec.java_compiler_debug"; + public static final OptionValidator JAVA_COMPILER_DEBUG = new BooleanValidator(JAVA_COMPILER_DEBUG_OPTION, true); + public static final String JAVA_COMPILER_JANINO_MAXSIZE_OPTION = "exec.java_compiler_janino_maxsize"; public static final OptionValidator JAVA_COMPILER_JANINO_MAXSIZE = new LongValidator(JAVA_COMPILER_JANINO_MAXSIZE_OPTION, 256*1024); + + public static final String JAVA_COMPILER_CONFIG = "drill.exec.compile.compiler"; + public static final String JAVA_COMPILER_DEBUG_CONFIG = "drill.exec.compile.debug"; + public static final String JAVA_COMPILER_JANINO_MAXSIZE_CONFIG = "drill.exec.compile.janino_maxsize"; private ClassCompilerSelector compilerSelector; @@ -64,9 +73,9 @@ public class QueryClassLoader extends URLClassLoader { private ConcurrentMap<String, byte[]> customClasses = new MapMaker().concurrencyLevel(4).makeMap(); - public QueryClassLoader(OptionManager sessionOptions) { + public QueryClassLoader(DrillConfig config, OptionManager sessionOptions) { super(new URL[0]); - compilerSelector = new ClassCompilerSelector(sessionOptions); + compilerSelector = new ClassCompilerSelector(config, sessionOptions); } public long getNextClassIndex(){ @@ -101,23 +110,27 @@ public class QueryClassLoader extends URLClassLoader { private final CompilerPolicy policy; private final long janinoThreshold; - private ClassCompiler jdkClassCompiler; - private ClassCompiler janinoClassCompiler; + private final AbstractClassCompiler jdkClassCompiler; + private final AbstractClassCompiler janinoClassCompiler; - ClassCompilerSelector(OptionManager sessionOptions) { + ClassCompilerSelector(DrillConfig config, OptionManager sessionOptions) { OptionValue value = sessionOptions.getOption(JAVA_COMPILER_OPTION); - this.policy = (value != null) ? CompilerPolicy.valueOf(value.string_val.toUpperCase()) : CompilerPolicy.DEFAULT; + this.policy = CompilerPolicy.valueOf((value != null) ? value.string_val.toUpperCase() : config.getString(JAVA_COMPILER_CONFIG).toUpperCase()); + value = sessionOptions.getOption(JAVA_COMPILER_JANINO_MAXSIZE_OPTION); - this.janinoThreshold = (value != null) ? value.num_val : JAVA_COMPILER_JANINO_MAXSIZE.getDefault().num_val; + this.janinoThreshold = (value != null) ? value.num_val : config.getLong(JAVA_COMPILER_JANINO_MAXSIZE_CONFIG); + + value = sessionOptions.getOption(JAVA_COMPILER_DEBUG_OPTION); + boolean debug = (value != null) ? value.bool_val : config.getBoolean(JAVA_COMPILER_DEBUG_CONFIG); - this.janinoClassCompiler = new JaninoClassCompiler(QueryClassLoader.this); - this.jdkClassCompiler = new JDKClassCompiler(QueryClassLoader.this); + this.janinoClassCompiler = (policy == CompilerPolicy.JANINO || policy == CompilerPolicy.DEFAULT) ? new JaninoClassCompiler(QueryClassLoader.this, debug) : null; + this.jdkClassCompiler = (policy == CompilerPolicy.JDK || policy == CompilerPolicy.DEFAULT) ? new JDKClassCompiler(QueryClassLoader.this, debug) : null; } private byte[][] getClassByteCode(ClassNames className, String sourceCode) throws CompileException, ClassNotFoundException, ClassTransformationException, IOException { - ClassCompiler classCompiler; + AbstractClassCompiler classCompiler; if (policy == CompilerPolicy.JDK || (policy == CompilerPolicy.DEFAULT && sourceCode.length() > janinoThreshold)) { classCompiler = jdkClassCompiler; } else { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java index 659a20c..c426918 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java @@ -100,7 +100,7 @@ public class FragmentContext implements Closeable { throw new ExecutionSetupException("Failure while reading plan options.", e); } this.allocator = context.getAllocator().getChildAllocator(fragment.getHandle(), fragment.getMemInitial(), fragment.getMemMax()); - this.loader = new QueryClassLoader(sessionOptions); + this.loader = new QueryClassLoader(dbContext.getConfig(), sessionOptions); } public OptionManager getOptions(){ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index bb45264..6a8cc5e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -73,6 +73,7 @@ public class SystemOptionManager implements OptionManager{ ExecConstants.MAX_HASH_TABLE_SIZE, QueryClassLoader.JAVA_COMPILER_VALIDATOR, QueryClassLoader.JAVA_COMPILER_JANINO_MAXSIZE, + QueryClassLoader.JAVA_COMPILER_DEBUG, ExecConstants.ENABLE_VERBOSE_ERRORS }; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/main/resources/drill-module.conf ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index 81a2bc2..f99c2fa 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -118,6 +118,11 @@ drill.exec: { size: 100000000 } }, + compile: { + compiler: "DEFAULT", + debug: true, + janino_maxsize: 262144 + }, sort: { purge.threshold : 100, external: { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java index beac02f..e1fefdf 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java @@ -17,24 +17,27 @@ */ package org.apache.drill.exec.compile; +import java.io.IOException; + import org.apache.drill.BaseTestQuery; -import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.compile.ClassTransformer.ClassSet; import org.apache.drill.exec.compile.sig.GeneratorMapping; import org.apache.drill.exec.compile.sig.MappingSet; +import org.apache.drill.exec.exception.ClassTransformationException; import org.apache.drill.exec.expr.ClassGenerator; import org.apache.drill.exec.expr.CodeGenerator; -import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; -import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.server.options.OptionValue.OptionType; import org.apache.drill.exec.server.options.SessionOptionManager; +import org.codehaus.commons.compiler.CompileException; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; public class TestClassTransformation extends BaseTestQuery { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestClassTransformation.class); - private static final int ITERATION_COUNT = 1; + private static final int ITERATION_COUNT = Integer.valueOf(System.getProperty("TestClassTransformation.iteration", "1")); private static SessionOptionManager sessionOptions; @@ -47,48 +50,83 @@ public class TestClassTransformation extends BaseTestQuery { public void testJaninoClassCompiler() throws Exception { logger.debug("Testing JaninoClassCompiler"); sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, QueryClassLoader.JAVA_COMPILER_OPTION, QueryClassLoader.CompilerPolicy.JANINO.name())); - QueryClassLoader loader = new QueryClassLoader(sessionOptions); + QueryClassLoader loader = new QueryClassLoader(config, sessionOptions); for (int i = 0; i < ITERATION_COUNT; i++) { compilationInnerClass(loader); } + loader.close(); } @Test public void testJDKClassCompiler() throws Exception { logger.debug("Testing JDKClassCompiler"); sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, QueryClassLoader.JAVA_COMPILER_OPTION, QueryClassLoader.CompilerPolicy.JDK.name())); - QueryClassLoader loader = new QueryClassLoader(sessionOptions); + QueryClassLoader loader = new QueryClassLoader(config, sessionOptions); for (int i = 0; i < ITERATION_COUNT; i++) { compilationInnerClass(loader); } + loader.close(); + } + + @Test + public void testCompilationNoDebug() throws CompileException, ClassNotFoundException, ClassTransformationException, IOException { + CodeGenerator<ExampleInner> cg = newCodeGenerator(ExampleInner.class, ExampleTemplateWithInner.class); + ClassSet classSet = new ClassSet(null, cg.getDefinition().getTemplateClassName(), cg.getMaterializedClassName()); + String sourceCode = cg.generate(); + sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, QueryClassLoader.JAVA_COMPILER_OPTION, QueryClassLoader.CompilerPolicy.JDK.name())); + + sessionOptions.setOption(OptionValue.createBoolean(OptionType.SESSION, QueryClassLoader.JAVA_COMPILER_DEBUG_OPTION, false)); + QueryClassLoader loader = new QueryClassLoader(config, sessionOptions); + final byte[][] codeWithoutDebug = loader.getClassByteCode(classSet.generated, sourceCode); + loader.close(); + int sizeWithoutDebug = 0; + for (byte[] bs : codeWithoutDebug) { + sizeWithoutDebug += bs.length; + } + + sessionOptions.setOption(OptionValue.createBoolean(OptionType.SESSION, QueryClassLoader.JAVA_COMPILER_DEBUG_OPTION, true)); + loader = new QueryClassLoader(config, sessionOptions); + final byte[][] codeWithDebug = loader.getClassByteCode(classSet.generated, sourceCode); + loader.close(); + int sizeWithDebug = 0; + for (byte[] bs : codeWithDebug) { + sizeWithDebug += bs.length; + } + + Assert.assertTrue("Debug code is smaller than optimized code!!!", sizeWithDebug > sizeWithoutDebug); + logger.debug("Optimized code is {}% smaller than debug code.", (int)((sizeWithDebug - sizeWithoutDebug)/(double)sizeWithDebug*100)); } /** * Do a test of a three level class to ensure that nested code generators works correctly. * @throws Exception */ - public void compilationInnerClass(QueryClassLoader loader) throws Exception{ - final TemplateClassDefinition<ExampleInner> template = new TemplateClassDefinition<>(ExampleInner.class, ExampleTemplateWithInner.class); - CodeGenerator<ExampleInner> cg = CodeGenerator.get(template, bit.getContext().getFunctionImplementationRegistry()); + private void compilationInnerClass(QueryClassLoader loader) throws Exception{ + CodeGenerator<ExampleInner> cg = newCodeGenerator(ExampleInner.class, ExampleTemplateWithInner.class); + + ClassTransformer ct = new ClassTransformer(); + ExampleInner t = ct.getImplementationClass(loader, cg.getDefinition(), cg.generate(), cg.getMaterializedClassName()); - ClassGenerator<ExampleInner> root = cg.getRoot(); + t.doOutside(); + t.doInsideOutside(); + } + + private <T, X extends T> CodeGenerator<T> newCodeGenerator(Class<T> iface, Class<X> impl) { + final TemplateClassDefinition<T> template = new TemplateClassDefinition<T>(iface, impl); + CodeGenerator<T> cg = CodeGenerator.get(template, bit.getContext().getFunctionImplementationRegistry()); + + ClassGenerator<T> root = cg.getRoot(); root.setMappingSet(new MappingSet(new GeneratorMapping("doOutside", null, null, null))); root.getSetupBlock().directStatement("System.out.println(\"outside\");"); - ClassGenerator<ExampleInner> inner = root.getInnerGenerator("TheInnerClass"); + ClassGenerator<T> inner = root.getInnerGenerator("TheInnerClass"); inner.setMappingSet(new MappingSet(new GeneratorMapping("doInside", null, null, null))); inner.getSetupBlock().directStatement("System.out.println(\"inside\");"); - ClassGenerator<ExampleInner> doubleInner = inner.getInnerGenerator("DoubleInner"); + ClassGenerator<T> doubleInner = inner.getInnerGenerator("DoubleInner"); doubleInner.setMappingSet(new MappingSet(new GeneratorMapping("doDouble", null, null, null))); doubleInner.getSetupBlock().directStatement("System.out.println(\"double\");"); - - ClassTransformer ct = new ClassTransformer(); - ExampleInner t = ct.getImplementationClass(loader, cg.getDefinition(), cg.generate(), cg.getMaterializedClassName()); - - t.doOutside(); - t.doInsideOutside(); + return cg; } - } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java index fecf66b..f6f06ad 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java @@ -28,7 +28,7 @@ public class TestLargeFileCompilation extends BaseTestQuery { private static final String LARGE_QUERY; - private static final int ITERATION_COUNT = 1; + private static final int ITERATION_COUNT = Integer.valueOf(System.getProperty("TestLargeFileCompilation.iteration", "1")); static { StringBuilder sb = new StringBuilder("select \n"); @@ -45,6 +45,12 @@ public class TestLargeFileCompilation extends BaseTestQuery { runTest(); } + @Test + public void testWithDEFAULT() throws Exception { + test(String.format("alter session set `%s`='DEFAULT'", QueryClassLoader.JAVA_COMPILER_OPTION)); + runTest(); + } + @Test(expected=org.apache.drill.exec.rpc.RpcException.class) public void testWithJanino() throws Exception { test(String.format("alter session set `%s`='JANINO'", QueryClassLoader.JAVA_COMPILER_OPTION)); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java index 328dd9c..62a224e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java @@ -82,6 +82,7 @@ public class TestNewMathFunctions { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; String planString = Resources.toString(Resources.getResource(planPath), Charsets.UTF_8); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java index 172bbed..6cb6ec1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java @@ -61,6 +61,7 @@ public class TestComparisonFunctions extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; String planString = Resources.toString(Resources.getResource(COMPARISON_TEST_PHYSICAL_PLAN), Charsets.UTF_8).replaceAll("EXPRESSION", expression); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java index ba3b646..3e4081d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java @@ -71,6 +71,7 @@ public class TestImplicitCastFunctions extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; String planString = Resources.toString(Resources.getResource(planPath), Charsets.UTF_8); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java index b032c51..18478a3 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java @@ -155,6 +155,7 @@ public class TestSimpleFunctions extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); @@ -195,6 +196,7 @@ public class TestSimpleFunctions extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); @@ -236,6 +238,7 @@ public class TestSimpleFunctions extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java index cb9e749..0f91366 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java @@ -74,6 +74,7 @@ public class TestStringFunctions extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; String planString = Resources.toString(Resources.getResource(planPath), Charsets.UTF_8); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java index e887584..e3b4daf 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java @@ -50,6 +50,7 @@ private SimpleRootExec doTest(final DrillbitContext bitContext, UserClientConnec bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java index 1f87371..2def4a6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java @@ -58,6 +58,7 @@ public class TestSimpleFilter extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; @@ -85,6 +86,7 @@ public class TestSimpleFilter extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java index 7afef56..65d51dc 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java @@ -76,6 +76,7 @@ public class TestMergeJoin extends PopUnitTestBase { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java index 0caf6d9..0d5d622 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java @@ -61,6 +61,7 @@ public class TestSimpleLimit extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; verifyLimitCount(bitContext, connection, "test1.json", 5); @@ -72,6 +73,7 @@ public class TestSimpleLimit extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; verifyLimitCount(bitContext, connection, "test3.json", 95); @@ -88,6 +90,7 @@ public class TestSimpleLimit extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; verifyLimitCount(bitContext, connection, "test2.json", 69999); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java index a05a632..63fb54b 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java @@ -66,6 +66,7 @@ public class TestSimpleProjection extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java index 6e9ecb1..f73889d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java @@ -63,6 +63,7 @@ public class TestSimpleSort extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; @@ -111,6 +112,7 @@ public class TestSimpleSort extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java index 6032681..e05a170 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java @@ -63,6 +63,7 @@ public class TestSVRemover extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java index 1fc2164..576f017 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java @@ -61,6 +61,7 @@ public class TestSimpleUnion extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; }}; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/97a9a4c1/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java ---------------------------------------------------------------------- diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java index 33a25b7..6ab7619 100644 --- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java +++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java @@ -39,7 +39,7 @@ import java.util.concurrent.TimeUnit; public class JdbcTestQueryBase extends JdbcTest { // Set a timeout unless we're debugging. @Rule - public TestRule TIMEOUT = TestTools.getTimeoutRule(20000); + public TestRule TIMEOUT = TestTools.getTimeoutRule(40000); protected static final String WORKING_PATH; static{
