DRILL-5355: Misc. code cleanup closes #784
Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/163a0c4e Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/163a0c4e Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/163a0c4e Branch: refs/heads/master Commit: 163a0c4e83d1500c27eb5d59d320f91e07961a0f Parents: 4c1017d Author: Paul Rogers <prog...@maprtech.com> Authored: Tue Mar 14 15:07:41 2017 -0700 Committer: Arina Ielchiieva <arina.yelchiy...@gmail.com> Committed: Fri Apr 7 15:24:51 2017 +0300 ---------------------------------------------------------------------- .../drill/exec/physical/impl/ScanBatch.java | 6 +- .../impl/flatten/FlattenRecordBatch.java | 2 + .../impl/project/ProjectorTemplate.java | 3 +- .../physical/visitor/PrelVisualizerVisitor.java | 228 +++++++++++++++++++ .../drill/exec/rpc/control/ControllerImpl.java | 1 + .../drill/exec/server/BootStrapContext.java | 9 +- .../drill/exec/store/dfs/FileSystemPlugin.java | 3 +- .../drill/exec/store/dfs/MagicString.java | 7 +- .../exec/store/dfs/easy/EasyFormatPlugin.java | 1 + .../exec/store/easy/json/JSONFormatPlugin.java | 3 - .../exec/store/easy/json/JSONRecordReader.java | 1 + .../compliant/CompliantTextRecordReader.java | 2 +- .../text/compliant/RepeatedVarCharOutput.java | 4 + .../store/sys/store/LocalPersistentStore.java | 3 + .../apache/drill/exec/util/TestUtilities.java | 2 +- .../exec/vector/complex/fn/JsonReader.java | 2 + .../complex/impl/VectorContainerWriter.java | 1 + .../exec/work/fragment/FragmentExecutor.java | 2 + .../java/org/apache/drill/PlanningBase.java | 4 +- .../drill/exec/DrillSeparatePlanningTest.java | 5 + .../physical/unit/BasicPhysicalOpUnitTest.java | 15 +- .../drill/exec/rpc/data/TestBitBitKerberos.java | 9 + .../drill/common/expression/FieldReference.java | 13 +- .../common/expression/LogicalExpression.java | 4 +- 24 files changed, 299 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java index e20c394..011e751 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java @@ -32,6 +32,7 @@ import org.apache.drill.common.types.Types; import org.apache.drill.exec.exception.OutOfMemoryException; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.expr.TypeHelper; +import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.ops.OperatorContext; import org.apache.drill.exec.physical.base.PhysicalOperator; @@ -84,6 +85,7 @@ public class ScanBatch implements CloseableRecordBatch { private Map<String, ValueVector> implicitVectors; private Iterator<Map<String, String>> implicitColumns; private Map<String, String> implicitValues; + private final BufferAllocator allocator; public ScanBatch(PhysicalOperator subScanConfig, FragmentContext context, OperatorContext oContext, Iterator<RecordReader> readers, @@ -95,6 +97,7 @@ public class ScanBatch implements CloseableRecordBatch { } currentReader = readers.next(); this.oContext = oContext; + allocator = oContext.getAllocator(); boolean setup = false; try { @@ -234,7 +237,6 @@ public class ScanBatch implements CloseableRecordBatch { w.getValueVector().getMutator().setValueCount(recordCount); } - // this is a slight misuse of this metric but it will allow Readers to report how many records they generated. final boolean isNewSchema = mutator.isNewSchema(); oContext.getStats().batchReceived(0, getRecordCount(), isNewSchema); @@ -335,7 +337,7 @@ public class ScanBatch implements CloseableRecordBatch { ValueVector v = fieldVectorMap.get(field.getPath()); if (v == null || v.getClass() != clazz) { // Field does not exist--add it to the map and the output container. - v = TypeHelper.getNewVector(field, oContext.getAllocator(), callBack); + v = TypeHelper.getNewVector(field, allocator, callBack); if (!clazz.isAssignableFrom(v.getClass())) { throw new SchemaChangeException( String.format( http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java index cadb3e3..fda00f8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java @@ -368,6 +368,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> { final MaterializedField outputField; if (expr instanceof ValueVectorReadExpression) { final TypedFieldId id = ValueVectorReadExpression.class.cast(expr).getFieldId(); + @SuppressWarnings("resource") final ValueVector incomingVector = incoming.getValueAccessorById(id.getIntermediateClass(), id.getFieldIds()).getValueVector(); // outputField is taken from the incoming schema to avoid the loss of nested fields // when the first batch will be empty. @@ -379,6 +380,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> { } else { outputField = MaterializedField.create(outputName, expr.getMajorType()); } + @SuppressWarnings("resource") final ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator()); allocationVectors.add(vector); TypedFieldId fid = container.add(vector); http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java index 9011e1f..499e55b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java @@ -105,7 +105,8 @@ public abstract class ProjectorTemplate implements Projector { @Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing) throws SchemaChangeException; - public abstract void doEval(@Named("inIndex") int inIndex, @Named("outIndex") int outIndex) + public abstract void doEval(@Named("inIndex") int inIndex, + @Named("outIndex") int outIndex) throws SchemaChangeException; } http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisualizerVisitor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisualizerVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisualizerVisitor.java new file mode 100644 index 0000000..703d648 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisualizerVisitor.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.physical.visitor; + +import org.apache.drill.exec.planner.physical.ExchangePrel; +import org.apache.drill.exec.planner.physical.JoinPrel; +import org.apache.drill.exec.planner.physical.Prel; +import org.apache.drill.exec.planner.physical.ProjectPrel; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.planner.physical.ScreenPrel; +import org.apache.drill.exec.planner.physical.WriterPrel; + +/** + * Debug-time class that prints a PRel tree to the console for + * inspection. Insert this into code during development to see + * the state of the tree at various points of interest during + * the planning process. + * <p> + * Use this by inserting lines into our prel transforms to see + * what is happening. This is useful if you must understand the transforms, + * or change them. For example: + * <p> + * In file: {@link DefaultSqlHandler#convertToPrel()}: + * <pre><code> + * PrelVisualizerVisitor.print("Before EER", phyRelNode); // Debug only + * phyRelNode = ExcessiveExchangeIdentifier.removeExcessiveEchanges(phyRelNode, targetSliceSize); + * PrelVisualizerVisitor.print("After EER", phyRelNode); // Debug only + * <code></pre> + */ + +public class PrelVisualizerVisitor + implements PrelVisitor<Void, PrelVisualizerVisitor.VisualizationState, Exception> { + + public static class VisualizationState { + + public static String INDENT = " "; + + StringBuilder out = new StringBuilder(); + int level; + + public void startNode(Prel prel) { + indent(); + out.append("{ "); + out.append(prel.getClass().getSimpleName()); + out.append("\n"); + push(); + } + + public void endNode() { + pop(); + indent(); + out.append("}"); + out.append("\n"); + } + + private void indent() { + for (int i = 0; i < level; i++) { + out.append(INDENT); + } + } + + public void push() { + level++; + } + + public void pop() { + level--; + } + + public void endFields() { + // TODO Auto-generated method stub + + } + + public void field(String label, boolean value) { + field(label, Boolean.toString(value)); + } + + private void field(String label, String value) { + indent(); + out.append(label) + .append(" = ") + .append(value) + .append("\n"); + } + + public void listField(String label, + Object[] values) { + if (values == null) { + field(label, "null"); + return; + } + StringBuilder buf = new StringBuilder(); + buf.append("["); + boolean first = true; + for (Object obj : values) { + if (! first) { + buf.append(", "); + } + first = false; + if (obj == null) { + buf.append("null"); + } else { + buf.append(obj.toString()); + } + } + buf.append("]"); + field(label, buf.toString()); + } + + @Override + public String toString() { + return out.toString(); + } + + } + + public static void print(String label, Prel prel) { + System.out.println(label); + System.out.println(visualize(prel)); + } + + public static String visualize(Prel prel) { + try { + VisualizationState state = new VisualizationState(); + prel.accept(new PrelVisualizerVisitor(), state); + return state.toString(); + } catch (Exception e) { + e.printStackTrace(); + return "** ERROR **"; + } + } + + @Override + public Void visitExchange(ExchangePrel prel, VisualizationState value) + throws Exception { + visitBasePrel(prel, value); + endNode(prel, value); + return null; + } + + private void visitBasePrel(Prel prel, VisualizationState value) { + value.startNode(prel); + value.listField("encodings", prel.getSupportedEncodings()); + value.field("needsReorder", prel.needsFinalColumnReordering()); + } + + private void endNode(Prel prel, VisualizationState value) throws Exception { + value.endFields(); + visitChildren(prel, value); + value.endNode(); + } + + private void visitChildren(Prel prel, VisualizationState value) throws Exception { + value.indent(); + value.out.append("children = [\n"); + value.push(); + for (Prel child : prel) { + child.accept(this, value); + } + value.pop(); + value.indent(); + value.out.append("]\n"); + } + + @Override + public Void visitScreen(ScreenPrel prel, VisualizationState value) + throws Exception { + visitBasePrel(prel, value); + endNode(prel, value); + return null; + } + + @Override + public Void visitWriter(WriterPrel prel, VisualizationState value) + throws Exception { + visitBasePrel(prel, value); + endNode(prel, value); + return null; + } + + @Override + public Void visitScan(ScanPrel prel, VisualizationState value) + throws Exception { + visitBasePrel(prel, value); + endNode(prel, value); + return null; + } + + @Override + public Void visitJoin(JoinPrel prel, VisualizationState value) + throws Exception { + visitBasePrel(prel, value); + endNode(prel, value); + return null; + } + + @Override + public Void visitProject(ProjectPrel prel, VisualizationState value) + throws Exception { + visitBasePrel(prel, value); + endNode(prel, value); + return null; + } + + @Override + public Void visitPrel(Prel prel, VisualizationState value) throws Exception { + visitBasePrel(prel, value); + endNode(prel, value); + return null; + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControllerImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControllerImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControllerImpl.java index 4991816..2bf5ad3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControllerImpl.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControllerImpl.java @@ -87,6 +87,7 @@ public class ControllerImpl implements Controller { } + @Override public void close() throws Exception { List<AutoCloseable> closeables = Lists.newArrayList(); closeables.add(server); http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java index 90ab018..ae68f3e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -47,7 +47,12 @@ import org.apache.hadoop.security.UserGroupInformation; public class BootStrapContext implements AutoCloseable { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BootStrapContext.class); - private static final int MIN_SCAN_THREADPOOL_SIZE = 8; // Magic num + // Tests and embedded servers need a small footprint, so the minimum + // scan count is small. The actual value is set by the + // ExecConstants.SCAN_THREADPOOL_SIZE. If the number below + // is large, then tests cannot shrink the number using the + // config property. + private static final int MIN_SCAN_THREADPOOL_SIZE = 4; // Magic num // DRILL_HOST_NAME sets custom host name. See drill-env.sh for details. private static final String customHostName = System.getenv("DRILL_HOST_NAME"); http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java index 7f2a9c1..cb3bfd1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java @@ -53,8 +53,7 @@ import com.google.common.collect.Maps; * Tables are file names, directories and path patterns. This storage engine delegates to FSFormatEngines but shares * references to the FileSystem configuration and path management. */ -public class FileSystemPlugin extends AbstractStoragePlugin{ - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemPlugin.class); +public class FileSystemPlugin extends AbstractStoragePlugin { private final FileSystemSchemaFactory schemaFactory; private final FormatCreator formatCreator; http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java index 0d4a0f5..838591f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,8 +18,6 @@ package org.apache.drill.exec.store.dfs; public class MagicString { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MagicString.class); - private long offset; private byte[] bytes; @@ -36,7 +34,4 @@ public class MagicString { public byte[] getBytes() { return bytes; } - - - } http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java index c09e009..776d806 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java @@ -126,6 +126,7 @@ public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements public abstract RecordReader getRecordReader(FragmentContext context, DrillFileSystem dfs, FileWork fileWork, List<SchemaPath> columns, String userName) throws ExecutionSetupException; + @SuppressWarnings("resource") CloseableRecordBatch getReaderBatch(FragmentContext context, EasySubScan scan) throws ExecutionSetupException { final ImplicitColumnExplorer columnExplorer = new ImplicitColumnExplorer(context, scan.getColumns()); http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java index 58ca95f..6f747ea 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java @@ -133,8 +133,6 @@ public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> { } return true; } - - } @Override @@ -151,5 +149,4 @@ public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> { public boolean supportsPushDown() { return true; } - } http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java index 7d929a1..ceb1deb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java @@ -149,6 +149,7 @@ public class JSONRecordReader extends AbstractRecordReader { } } + @Override protected List<SchemaPath> getDefaultColumnsToRead() { return ImmutableList.of(); } http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/CompliantTextRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/CompliantTextRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/CompliantTextRecordReader.java index ba01bc9..93c4ff8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/CompliantTextRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/CompliantTextRecordReader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java index 2ec662e..eda2feb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java @@ -172,6 +172,7 @@ class RepeatedVarCharOutput extends TextOutput { * Start a new record batch. Resets all the offsets and pointers that * store buffer addresses */ + @Override public void startBatch() { this.recordStart = characterDataOriginal; this.fieldOpen = false; @@ -185,6 +186,7 @@ class RepeatedVarCharOutput extends TextOutput { } private void loadRepeatedOffsetAddress(){ + @SuppressWarnings("resource") DrillBuf buf = vector.getOffsetVector().getBuffer(); checkBuf(buf); this.repeatedOffset = buf.memoryAddress() + 4; @@ -193,6 +195,7 @@ class RepeatedVarCharOutput extends TextOutput { } private void loadVarCharDataAddress(){ + @SuppressWarnings("resource") DrillBuf buf = vector.getDataVector().getBuffer(); checkBuf(buf); this.characterData = buf.memoryAddress(); @@ -201,6 +204,7 @@ class RepeatedVarCharOutput extends TextOutput { } private void loadVarCharOffsetAddress(){ + @SuppressWarnings("resource") DrillBuf buf = vector.getDataVector().getOffsetVector().getBuffer(); checkBuf(buf); this.charLengthOffset = buf.memoryAddress() + 4; http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java index ef855e2..dc4c414 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java @@ -89,6 +89,9 @@ public class LocalPersistentStore<V> extends BasePersistentStore<V> { public static Path getLogDir() { String drillLogDir = System.getenv("DRILL_LOG_DIR"); if (drillLogDir == null) { + drillLogDir = System.getProperty("drill.log.dir"); + } + if (drillLogDir == null) { drillLogDir = "/var/log/drill"; } return new Path(new File(drillLogDir).getAbsoluteFile().toURI()); http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/util/TestUtilities.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/TestUtilities.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TestUtilities.java index 5498ad4..3532956 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/TestUtilities.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TestUtilities.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java index 9848ae7..cfad551 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java @@ -101,6 +101,7 @@ public class JsonReader extends BaseJsonProcessor { this.readNumbersAsDouble = readNumbersAsDouble; } + @SuppressWarnings("resource") @Override public void ensureAtLeastOneField(ComplexWriter writer) { List<BaseWriter.MapWriter> writerList = Lists.newArrayList(); @@ -192,6 +193,7 @@ public class JsonReader extends BaseJsonProcessor { setSource(data.getBytes(Charsets.UTF_8)); } + @SuppressWarnings("resource") public void setSource(byte[] bytes) throws IOException { setSource(new SeekableBAIS(bytes)); } http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java index b19b029..5d3f9ce 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java @@ -101,6 +101,7 @@ public class VectorContainerWriter extends AbstractFieldWriter implements Comple super("", null, callback); } + @SuppressWarnings("resource") @Override public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) { try { http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java index b167fa3..c57869a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java @@ -179,6 +179,7 @@ public class FragmentExecutor implements Runnable { eventProcessor.receiverFinished(handle); } + @SuppressWarnings("resource") @Override public void run() { // if a cancel thread has already entered this executor, we have not reason to continue. @@ -224,6 +225,7 @@ public class FragmentExecutor implements Runnable { ImpersonationUtil.getProcessUserUGI(); queryUserUgi.doAs(new PrivilegedExceptionAction<Void>() { + @Override public Void run() throws Exception { injector.injectChecked(fragmentContext.getExecutionControls(), "fragment-execution", IOException.class); /* http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java index ad9cc64..df0c89f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -82,6 +82,7 @@ public class PlanningBase extends ExecTest{ final LogicalPlanPersistence logicalPlanPersistence = new LogicalPlanPersistence(config, scanResult); final SystemOptionManager systemOptions = new SystemOptionManager(logicalPlanPersistence , provider); systemOptions.init(); + @SuppressWarnings("resource") final UserSession userSession = UserSession.Builder.newBuilder().withOptionManager(systemOptions).build(); final SessionOptionManager sessionOptions = (SessionOptionManager) userSession.getOptions(); final QueryOptionManager queryOptions = new QueryOptionManager(sessionOptions); @@ -150,6 +151,7 @@ public class PlanningBase extends ExecTest{ if (sql.trim().isEmpty()) { continue; } + @SuppressWarnings("unused") final PhysicalPlan p = DrillSqlWorker.getPlan(context, sql); } } http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSeparatePlanningTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSeparatePlanningTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSeparatePlanningTest.java index 0481825..58e94e3 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSeparatePlanningTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSeparatePlanningTest.java @@ -186,6 +186,7 @@ public class DrillSeparatePlanningTest extends BaseTestQuery { //AwaitableUserResultsListener listener = // new AwaitableUserResultsListener(new SilentListener()); client.runQuery(QueryType.SQL, query, listener); + @SuppressWarnings("unused") int rows = listener.await(); } @@ -211,6 +212,7 @@ public class DrillSeparatePlanningTest extends BaseTestQuery { private void getResultsHelper(final QueryPlanFragments planFragments) throws Exception { for (PlanFragment fragment : planFragments.getFragmentsList()) { DrillbitEndpoint assignedNode = fragment.getAssignment(); + @SuppressWarnings("resource") DrillClient fragmentClient = new DrillClient(true); Properties props = new Properties(); props.setProperty("drillbit", assignedNode.getAddress() + ":" + assignedNode.getUserPort()); @@ -250,6 +252,7 @@ public class DrillSeparatePlanningTest extends BaseTestQuery { AwaitableUserResultsListener listener = new AwaitableUserResultsListener(new SilentListener()); fragmentClient.runQuery(QueryType.EXECUTION, fragmentList, listener); + @SuppressWarnings("unused") int rows = listener.await(); fragmentClient.close(); } @@ -257,6 +260,7 @@ public class DrillSeparatePlanningTest extends BaseTestQuery { private void getCombinedResultsHelper(final QueryPlanFragments planFragments) throws Exception { ShowResultsUserResultsListener myListener = new ShowResultsUserResultsListener(getAllocator()); + @SuppressWarnings("unused") AwaitableUserResultsListener listenerBits = new AwaitableUserResultsListener(myListener); @@ -265,6 +269,7 @@ public class DrillSeparatePlanningTest extends BaseTestQuery { AwaitableUserResultsListener listener = new AwaitableUserResultsListener(new SilentListener()); client.runQuery(QueryType.EXECUTION, planFragments.getFragmentsList(), listener); + @SuppressWarnings("unused") int rows = listener.await(); } http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/BasicPhysicalOpUnitTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/BasicPhysicalOpUnitTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/BasicPhysicalOpUnitTest.java index 6f2f160..a0ea6fe 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/BasicPhysicalOpUnitTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/BasicPhysicalOpUnitTest.java @@ -17,13 +17,13 @@ */ package org.apache.drill.exec.physical.unit; -import com.google.common.collect.Lists; +import static org.apache.drill.TestBuilder.mapOf; + +import java.util.List; + import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.core.JoinRelType; import org.apache.drill.exec.physical.MinorFragmentEndpoint; -import org.apache.drill.exec.physical.base.GroupScan; -import org.apache.drill.exec.physical.base.PhysicalOperator; -import org.apache.drill.exec.physical.base.SubScan; import org.apache.drill.exec.physical.config.ComplexToJson; import org.apache.drill.exec.physical.config.ExternalSort; import org.apache.drill.exec.physical.config.Filter; @@ -37,12 +37,7 @@ import org.apache.drill.exec.physical.config.TopN; import org.junit.Ignore; import org.junit.Test; -import java.lang.reflect.Constructor; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.Set; - -import static org.apache.drill.TestBuilder.mapOf; +import com.google.common.collect.Lists; public class BasicPhysicalOpUnitTest extends PhysicalOpUnitTestBase { http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/data/TestBitBitKerberos.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/data/TestBitBitKerberos.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/data/TestBitBitKerberos.java index ffcacc9..ce43783 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/data/TestBitBitKerberos.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/data/TestBitBitKerberos.java @@ -95,6 +95,7 @@ public class TestBitBitKerberos extends BaseTestQuery { private static boolean kdcStarted; + @SuppressWarnings("restriction") @BeforeClass public static void setupKdc() throws Exception { kdc = new SimpleKdcServer(); @@ -159,6 +160,7 @@ public class TestBitBitKerberos extends BaseTestQuery { } private static int getFreePort() throws IOException { + @SuppressWarnings("resource") ServerSocket s = null; try { s = new ServerSocket(0); @@ -209,10 +211,13 @@ public class TestBitBitKerberos extends BaseTestQuery { public void success(@Injectable WorkerBee bee, @Injectable final WorkEventBus workBus) throws Exception { ScanResult result = ClassPathScanner.fromPrescan(newConfig); + @SuppressWarnings("resource") final BootStrapContext c1 = new BootStrapContext(newConfig, result); + @SuppressWarnings({ "unused", "resource" }) final BootStrapContext c2 = new BootStrapContext(newConfig, result); final FragmentContext fcontext = new MockUp<FragmentContext>(){ + @SuppressWarnings("unused") BufferAllocator getAllocator(){ return c1.getAllocator(); } @@ -239,6 +244,7 @@ public class TestBitBitKerberos extends BaseTestQuery { return true; } + @SuppressWarnings("unused") public FragmentContext getFragmentContext(){ return fcontext; } @@ -255,10 +261,12 @@ public class TestBitBitKerberos extends BaseTestQuery { DataConnectionConfig config = new DataConnectionConfig(c1.getAllocator(), c1, new DataServerRequestHandler(workBus, bee)); + @SuppressWarnings("resource") DataServer server = new DataServer(config); port = server.bind(port, true); DrillbitEndpoint ep = DrillbitEndpoint.newBuilder().setAddress("localhost").setDataPort(port).build(); + @SuppressWarnings("resource") DataConnectionManager connectionManager = new DataConnectionManager(ep, config); DataTunnel tunnel = new DataTunnel(connectionManager); AtomicLong max = new AtomicLong(0); @@ -277,6 +285,7 @@ public class TestBitBitKerberos extends BaseTestQuery { private static WritableBatch getRandomBatch(BufferAllocator allocator, int records) { List<ValueVector> vectors = Lists.newArrayList(); for (int i = 0; i < 5; i++) { + @SuppressWarnings("resource") Float8Vector v = (Float8Vector) TypeHelper.getNewVector( MaterializedField.create("a", Types.required(MinorType.FLOAT8)), allocator); http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/logical/src/main/java/org/apache/drill/common/expression/FieldReference.java ---------------------------------------------------------------------- diff --git a/logical/src/main/java/org/apache/drill/common/expression/FieldReference.java b/logical/src/main/java/org/apache/drill/common/expression/FieldReference.java index 7d0e86f..640984e 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/FieldReference.java +++ b/logical/src/main/java/org/apache/drill/common/expression/FieldReference.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -68,6 +68,15 @@ public class FieldReference extends SchemaPath { checkSimpleString(value); } + /** + * Create a {@link FieldReference} given an unquoted name. (Note: the + * name here is a misnomer: the name may have been quoted in SQL, but + * must be unquoted when passed in here.) + * + * @param safeString the unquoted field reference + * @return the field reference expression + */ + public static FieldReference getWithQuotedRef(CharSequence safeString) { return new FieldReference(safeString, ExpressionPosition.UNKNOWN, false); } @@ -100,6 +109,7 @@ public class FieldReference extends SchemaPath { } } + @SuppressWarnings("serial") public static class De extends StdDeserializer<FieldReference> { public De() { @@ -116,6 +126,7 @@ public class FieldReference extends SchemaPath { } + @SuppressWarnings("serial") public static class Se extends StdSerializer<FieldReference> { public Se() { http://git-wip-us.apache.org/repos/asf/drill/blob/163a0c4e/logical/src/main/java/org/apache/drill/common/expression/LogicalExpression.java ---------------------------------------------------------------------- diff --git a/logical/src/main/java/org/apache/drill/common/expression/LogicalExpression.java b/logical/src/main/java/org/apache/drill/common/expression/LogicalExpression.java index a74ae29..7b6001d 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/LogicalExpression.java +++ b/logical/src/main/java/org/apache/drill/common/expression/LogicalExpression.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -55,6 +55,7 @@ public interface LogicalExpression extends Iterable<LogicalExpression>{ public int getSelfCost(); public int getCumulativeCost(); + @SuppressWarnings("serial") public static class De extends StdDeserializer<LogicalExpression> { DrillConfig config; @@ -90,6 +91,7 @@ public interface LogicalExpression extends Iterable<LogicalExpression>{ } + @SuppressWarnings("serial") public static class Se extends StdSerializer<LogicalExpression> { protected Se() {
