DRILL-925: fix issues with dynamic memory allocation
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ed260f1d Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ed260f1d Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ed260f1d Branch: refs/heads/master Commit: ed260f1d023d23dc78c7c98f0dd40d9d0e021750 Parents: 2dec152 Author: Steven Phillips <[email protected]> Authored: Sat May 31 17:47:24 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Sun Jun 8 19:13:05 2014 -0700 ---------------------------------------------------------------------- .../codegen/templates/FixedValueVectors.java | 60 ++++++++++++++++---- .../templates/VariableLengthVectors.java | 10 +++- .../drill/exec/record/RecordBatchLoader.java | 1 + 3 files changed, 57 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed260f1d/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java index 46e3c59..22b92a6 100644 --- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java @@ -205,7 +205,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ - if(thisIndex >= getValueCapacity()) return false; + if(thisIndex >= getValueCapacity()) { + allocationMonitor--; + return false; + } copyFrom(fromIndex, thisIndex, from); return true; } @@ -586,13 +589,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } public boolean setSafe(int index, ${minor.class}Holder holder){ - if(index >= getValueCapacity()) return false; + if(index >= getValueCapacity()) { + allocationMonitor--; + return false; + } set(index, holder); return true; } public boolean setSafe(int index, Nullable${minor.class}Holder holder){ - if(index >= getValueCapacity()) return false; + if(index >= getValueCapacity()) { + allocationMonitor--; + return false; + } set(index, holder); return true; } @@ -612,13 +621,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } public boolean setSafe(int index, ${minor.class}Holder holder){ - if(index >= getValueCapacity()) return false; + if(index >= getValueCapacity()) { + allocationMonitor--; + return false; + } set(index, holder); return true; } public boolean setSafe(int index, Nullable${minor.class}Holder holder){ - if(index >= getValueCapacity()) return false; + if(index >= getValueCapacity()) { + allocationMonitor--; + return false; + } set(index, holder); return true; } @@ -636,13 +651,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } public boolean setSafe(int index, ${minor.class}Holder holder){ - if(index >= getValueCapacity()) return false; + if(index >= getValueCapacity()) { + allocationMonitor--; + return false; + } set(index, holder); return true; } public boolean setSafe(int index, Nullable${minor.class}Holder holder){ - if(index >= getValueCapacity()) return false; + if(index >= getValueCapacity()) { + allocationMonitor--; + return false; + } set(index, holder); return true; } @@ -679,13 +700,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } public boolean setSafe(int index, Nullable${minor.class}Holder holder){ - if(index >= getValueCapacity()) return false; + if(index >= getValueCapacity()) { + allocationMonitor--; + return false; + } set(index, holder); return true; } public boolean setSafe(int index, ${minor.class}Holder holder){ - if(index >= getValueCapacity()) return false; + if(index >= getValueCapacity()) { + allocationMonitor--; + return false; + } set(index, holder); return true; } @@ -696,7 +723,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } public boolean setSafe(int index, ${minor.class}Holder holder){ - if(index >= getValueCapacity()) return false; + if(index >= getValueCapacity()) { + allocationMonitor--; + return false; + } set(index, holder); return true; } @@ -737,7 +767,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } public boolean setSafe(int index, ${minor.class}Holder holder){ - if(index >= getValueCapacity()) return false; + if(index >= getValueCapacity()) { + allocationMonitor--; + return false; + } set(index, holder); return true; } @@ -747,7 +780,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } public boolean setSafe(int index, Nullable${minor.class}Holder holder){ - if(index >= getValueCapacity()) return false; + if(index >= getValueCapacity()) { + allocationMonitor--; + return false; + } set(index, holder); return true; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed260f1d/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java index cc6f5e7..475433c 100644 --- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java +++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java @@ -167,7 +167,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ - if(thisIndex >= getValueCapacity()) return false; + if(thisIndex >= getValueCapacity()) { + allocationMonitor--; + return false; + } int start = from.offsetVector.getAccessor().get(fromIndex); int end = from.offsetVector.getAccessor().get(fromIndex+1); @@ -175,7 +178,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width}); - if(data.capacity() < outputStart + len) return false; + if(data.capacity() < outputStart + len) { + allocationMonitor--; + return false; + } from.data.getBytes(start, data, outputStart, len); offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed260f1d/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java index e32fda9..cac610c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java @@ -60,6 +60,7 @@ public class RecordBatchLoader implements VectorAccessible, Iterable<VectorWrapp */ public boolean load(RecordBatchDef def, ByteBuf buf) throws SchemaChangeException { // logger.debug("Loading record batch with def {} and data {}", def, buf); + container.zeroVectors(); this.valueCount = def.getRecordCount(); boolean schemaChanged = schema == null; // logger.info("Load, ThreadID: {}", Thread.currentThread().getId(), new RuntimeException("For Stack Trace Only"));
