date:20180407

[01/11] drill git commit: DRILL-6279: Indicate operators that spilled in-memory data to disk on Web UI

2018-04-07 Thread arina

Repository: drill
Updated Branches:
  refs/heads/master 9a6cb59b9 -> da241134f


DRILL-6279: Indicate operators that spilled in-memory data to disk on Web UI

As part of this coomit added Bootstrap's Glyphicons.

closes #1197


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/cc440af0
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/cc440af0
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/cc440af0

Branch: refs/heads/master
Commit: cc440af0397d0ec8b52227db9b7e09513b5ff157
Parents: 9a6cb59
Author: Kunal Khatua 
Authored: Sun Mar 25 22:05:06 2018 -0700
Committer: Arina Ielchiieva 
Committed: Fri Apr 6 11:52:22 2018 +0300

--
 .../server/rest/profile/OperatorWrapper.java|  89 ++-
 .../exec/server/rest/profile/TableBuilder.java  |  14 ++-
 .../src/main/resources/rest/profile/profile.ftl |  12 ++-
 .../fonts/glyphicons-halflings-regular.eot  | Bin 0 -> 20127 bytes
 .../fonts/glyphicons-halflings-regular.ttf  | Bin 0 -> 45404 bytes
 .../fonts/glyphicons-halflings-regular.woff | Bin 0 -> 23424 bytes
 .../fonts/glyphicons-halflings-regular.woff2| Bin 0 -> 18028 bytes
 7 files changed, 109 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/cc440af0/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
index afccbb7..3a60c71 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
@@ -17,6 +17,7 @@
  */
 package org.apache.drill.exec.server.rest.profile;
 
+import java.text.DecimalFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -38,7 +39,17 @@ import com.google.common.base.Preconditions;
  * Wrapper class for profiles of ALL operator instances of the same operator 
type within a major fragment.
  */
 public class OperatorWrapper {
+  @SuppressWarnings("unused")
+  private static final org.slf4j.Logger logger = 
org.slf4j.LoggerFactory.getLogger(OperatorWrapper.class);
+
+  private static final String HTML_ATTRIB_SPILLS = "spills";
+  private static final String HTML_ATTRIB_CLASS = "class";
+  private static final String HTML_ATTRIB_STYLE = "style";
+  private static final String HTML_ATTRIB_TITLE = "title";
+  private static final DecimalFormat DECIMAL_FORMATTER = new 
DecimalFormat("#.##");
   private static final String UNKNOWN_OPERATOR = "UNKNOWN_OPERATOR";
+  //Negative valued constant used for denoting invalid index to indicate 
absence of metric
+  private static final int NO_SPILL_METRIC_INDEX = Integer.MIN_VALUE;
   private final int major;
   private final List, 
String>> opsAndHosts; // [(operatorProfile --> minorFragment number,host), ...]
   private final OperatorProfile firstProfile;
@@ -146,6 +157,11 @@ public class OperatorWrapper {
 tb.appendCell(path, null, null, opTblBgColor);
 tb.appendCell(operatorName);
 
+//Check if spill information is available
+int spillCycleMetricIndex = getSpillCycleMetricIndex(operatorType);
+boolean isSpillableOp = (spillCycleMetricIndex != NO_SPILL_METRIC_INDEX);
+boolean hasSpilledToDisk = false;
+
 //Get MajorFragment Busy+Wait Time Tally
 long majorBusyNanos = majorFragmentBusyTally.get(new 
OperatorPathBuilder().setMajor(major).build());
 
@@ -153,6 +169,8 @@ public class OperatorWrapper {
 double processSum = 0.0;
 double waitSum = 0.0;
 double memSum = 0.0;
+double spillCycleSum = 0.0;
+long spillCycleMax = 0L;
 long recordSum = 0L;
 
 //Construct list for sorting purposes (using legacy Comparators)
@@ -168,6 +186,22 @@ public class OperatorWrapper {
 recordSum += sp.getRecords();
   }
   opList.add(ip.getLeft());
+
+  //Capture Spill Info
+  //Check to ensure index < #metrics (old profiles have less metrics); 
else reset isSpillableOp
+  if (isSpillableOp) {
+//NOTE: We get non-zero value for non-existent metrics, so we can't 
use getMetric(index)
+//profile.getMetric(spillCycleMetricIndex).getLongValue();
+//Forced to iterate list
+for (MetricValue metricVal : profile.getMetricList()) {
+  if (metricVal.getMetricId() == spillCycleMetricIndex) {
+long spillCycles = metricVal.getLongValue();
+spillCycleMax =

[06/11] drill git commit: DRILL-6230: Extend row set readers to handle hyper vectors

2018-04-07 Thread arina

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseElementReader.java
--
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseElementReader.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseElementReader.java
deleted file mode 100644
index f32c101..000
--- 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseElementReader.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.accessor.reader;
-
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
-import org.apache.drill.exec.vector.accessor.ObjectType;
-import org.apache.drill.exec.vector.accessor.ScalarElementReader;
-import org.apache.drill.exec.vector.accessor.impl.AccessorUtilities;
-import org.joda.time.Period;
-
-public abstract class BaseElementReader implements ScalarElementReader {
-
-  public static class ScalarElementObjectReader extends AbstractObjectReader {
-
-private BaseElementReader elementReader;
-
-public ScalarElementObjectReader(BaseElementReader elementReader) {
-  this.elementReader = elementReader;
-}
-
-@Override
-public void bindIndex(ColumnReaderIndex index) {
-  elementReader.bindIndex((ElementReaderIndex) index);
-}
-
-@Override
-public ObjectType type() {
-  return ObjectType.SCALAR;
-}
-
-@Override
-public ScalarElementReader elements() {
-  return elementReader;
-}
-
-@Override
-public Object getObject() {
-  // Simple: return elements as an object list.
-  // If really needed, could return as a typed array, but that
-  // is a bit of a hassle.
-
-  List elements = new ArrayList<>();
-  for (int i = 0; i < elementReader.size(); i++) {
-elements.add(elementReader.getObject(i));
-  }
-  return elements;
-}
-
-@Override
-public String getAsString() {
-  StringBuilder buf = new StringBuilder();
-  buf.append("[");
-  for (int i = 0; i < elementReader.size(); i++) {
-if (i > 0) {
-  buf.append( ", " );
-}
-buf.append(elementReader.getAsString(i));
-  }
-  buf.append("]");
-  return buf.toString();
-}
-  }
-
-  protected ElementReaderIndex vectorIndex;
-  protected VectorAccessor vectorAccessor;
-
-  public abstract void bindVector(ValueVector vector);
-
-  public void bindVector(MajorType majorType, VectorAccessor va) {
-vectorAccessor = va;
-  }
-
-  protected void bindIndex(ElementReaderIndex rowIndex) {
-this.vectorIndex = rowIndex;
-  }
-
-  @Override
-  public int size() { return vectorIndex.size(); }
-
-  @Override
-  public Object getObject(int index) {
-if (isNull(index)) {
-  return "null";
-}
-switch (valueType()) {
-case BYTES:
-  return getBytes(index);
-case DECIMAL:
-  return getDecimal(index);
-case DOUBLE:
-  return getDouble(index);
-case INTEGER:
-  return getInt(index);
-case LONG:
-  return getLong(index);
-case PERIOD:
-  return getPeriod(index);
-case STRING:
-  return getString(index);
-default:
-  throw new IllegalStateException("Unexpected type: " + valueType());
-}
-  }
-
-  @Override
-  public String getAsString(int index) {
-switch (valueType()) {
-case BYTES:
-  return AccessorUtilities.bytesToString(getBytes(index));
-case DOUBLE:
-  return Double.toString(getDouble(index));
-case INTEGER:
-  return Integer.toString(getInt(index));
-case LONG:
-  return Long.toString(getLong(index));
-case STRING:
-  return "\"" + getString(index) + "\"";
-case DECIMAL:
-  return getDecimal(index).toPlainString();
-case PERIOD:
-  return getPeriod(index).normalizedStandard().toString();
-default:
-  throw new

[09/11] drill git commit: DRILL-6230: Extend row set readers to handle hyper vectors

2018-04-07 Thread arina

DRILL-6230: Extend row set readers to handle hyper vectors

closes #1161


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4f2182e4
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4f2182e4
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4f2182e4

Branch: refs/heads/master
Commit: 4f2182e41f4474ca42ae6d572a9c5d5ff274d984
Parents: 127e415
Author: Paul Rogers 
Authored: Sat Mar 10 23:43:36 2018 -0800
Committer: Arina Ielchiieva 
Committed: Fri Apr 6 12:05:14 2018 +0300

--
 .../rowSet/model/AbstractReaderBuilder.java |  45 ++
 .../exec/physical/rowSet/model/ReaderIndex.java |  28 +-
 .../physical/rowSet/model/SchemaInference.java  |  62 ---
 .../rowSet/model/hyper/BaseReaderBuilder.java   | 142 --
 .../model/hyper/HyperSchemaInference.java   |  72 +++
 .../rowSet/model/single/BaseReaderBuilder.java  |  81 ++--
 .../rowSet/model/single/DirectRowIndex.java |  40 ++
 .../model/single/SingleSchemaInference.java |  65 +++
 .../physical/rowSet/impl/RowSetTestUtils.java   |  59 +++
 .../impl/TestResultSetLoaderMapArray.java   | 135 +++---
 .../rowSet/impl/TestResultSetLoaderMaps.java| 193 +---
 .../impl/TestResultSetLoaderOmittedValues.java  |   3 +-
 .../impl/TestResultSetLoaderOverflow.java   | 173 ---
 .../impl/TestResultSetLoaderProjection.java | 139 +-
 .../impl/TestResultSetLoaderProtocol.java   |   2 +-
 .../rowSet/impl/TestResultSetLoaderTorture.java |  16 +-
 .../drill/exec/record/TestVectorContainer.java  |   3 +-
 .../apache/drill/test/rowSet/DirectRowSet.java  |  31 +-
 .../drill/test/rowSet/HyperRowSetImpl.java  | 102 +++-
 .../drill/test/rowSet/IndirectRowSet.java   |  14 +-
 .../org/apache/drill/test/rowSet/RowSet.java|   7 +
 .../drill/test/rowSet/RowSetComparison.java |  73 +--
 .../apache/drill/test/rowSet/RowSetPrinter.java |  18 +-
 .../apache/drill/test/rowSet/RowSetReader.java  |  11 +-
 .../drill/test/rowSet/RowSetReaderImpl.java |  24 +-
 .../drill/test/rowSet/schema/UnionBuilder.java  |   4 +-
 .../drill/test/rowSet/test/RowSetTest.java  | 268 +--
 .../drill/test/rowSet/test/TestFillEmpties.java |  12 +-
 .../rowSet/test/TestHyperVectorReaders.java | 365 ++
 .../test/rowSet/test/TestScalarAccessors.java   | 474 +++
 .../drill/test/rowSet/test/VectorPrinter.java   |  72 ---
 .../src/main/java/io/netty/buffer/DrillBuf.java |  15 +
 .../main/codegen/templates/ColumnAccessors.java | 251 +-
 .../drill/exec/vector/accessor/ArrayReader.java |  40 +-
 .../exec/vector/accessor/ColumnReader.java  |  83 
 .../exec/vector/accessor/ColumnReaderIndex.java | 102 +++-
 .../exec/vector/accessor/ObjectReader.java  |  26 +-
 .../vector/accessor/ScalarElementReader.java|  65 ---
 .../exec/vector/accessor/ScalarReader.java  |  13 +-
 .../drill/exec/vector/accessor/TupleReader.java |  31 +-
 .../accessor/UnsupportedConversionError.java|  52 ++
 .../drill/exec/vector/accessor/ValueType.java   |  57 ++-
 .../vector/accessor/impl/VectorPrinter.java |  72 +++
 .../exec/vector/accessor/package-info.java  | 184 +++
 .../accessor/reader/AbstractArrayReader.java| 188 
 .../accessor/reader/AbstractObjectReader.java   |  21 +-
 .../accessor/reader/AbstractScalarReader.java   | 205 
 .../accessor/reader/AbstractTupleReader.java|  85 ++--
 .../vector/accessor/reader/ArrayReaderImpl.java | 357 ++
 .../accessor/reader/BaseElementReader.java  | 187 
 .../accessor/reader/BaseScalarReader.java   | 198 
 .../accessor/reader/ColumnReaderFactory.java|  52 +-
 .../reader/FixedWidthElementReaderIndex.java|  38 --
 .../exec/vector/accessor/reader/MapReader.java  |  49 +-
 .../vector/accessor/reader/NullStateReader.java |  52 ++
 .../accessor/reader/NullStateReaders.java   | 193 
 .../accessor/reader/ObjectArrayReader.java  | 159 ---
 .../accessor/reader/OffsetVectorReader.java |  70 +++
 .../vector/accessor/reader/ReaderEvents.java|  31 ++
 .../accessor/reader/ScalarArrayReader.java  | 102 
 .../vector/accessor/reader/VectorAccessor.java  |   5 +-
 .../vector/accessor/reader/VectorAccessors.java | 344 ++
 .../vector/accessor/reader/package-info.java|  65 ++-
 .../accessor/writer/ColumnWriterFactory.java|   4 +-
 .../drill/exec/vector/complex/ListVector.java   |   2 +-
 65 files changed, 4122 insertions(+), 2009 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/model/AbstractReaderBuilder.java
--
diff --git

[03/11] drill git commit: DRILL-6287: apache-release profile should be disabled by default

2018-04-07 Thread arina

DRILL-6287: apache-release profile should be disabled by default

closes #1182


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a0549453
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a0549453
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a0549453

Branch: refs/heads/master
Commit: a05494537494efe7d9489e1468245e1ca7b16708
Parents: 2e1568c
Author: Vlad Rozov 
Authored: Thu Mar 22 06:54:56 2018 -0700
Committer: Arina Ielchiieva 
Committed: Fri Apr 6 11:56:55 2018 +0300

--
 pom.xml | 11 ---
 1 file changed, 4 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/a0549453/pom.xml
--
diff --git a/pom.xml b/pom.xml
index f07beaf..b9d395d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -66,6 +66,7 @@
 
 4096
 4096
+-Xdoclint:none
   
 
   
@@ -1234,12 +1235,6 @@
   
   apache-release
-  
-[1.8,)
-  
-  
--Xdoclint:none
-  
   
 
   
@@ -1274,7 +1269,9 @@
 
   true
   false
-  
src/main/resources/assemblies/source-assembly.xml
+  
+
src/main/resources/assemblies/source-assembly.xml
+  
   apache-drill-${project.version}-src
   gnu

[11/11] drill git commit: DRILL-6296: Add operator metrics for batch sizing for merge join

2018-04-07 Thread arina

DRILL-6296: Add operator metrics for batch sizing for merge join

closes #1181


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/da241134
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/da241134
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/da241134

Branch: refs/heads/master
Commit: da241134fb88464139437b05b1feaafbb3014bb0
Parents: 77f5e90
Author: Padma Penumarthy 
Authored: Thu Mar 29 19:05:07 2018 -0700
Committer: Arina Ielchiieva 
Committed: Fri Apr 6 12:07:30 2018 +0300

--
 .../drill/exec/ops/OperatorMetricRegistry.java  |   2 +
 .../impl/flatten/FlattenRecordBatch.java|  28 ++-
 .../exec/physical/impl/join/MergeJoinBatch.java |  90 +++-
 .../AbstractRecordBatchMemoryManager.java   | 134 ---
 .../exec/record/RecordBatchMemoryManager.java   | 228 +++
 .../drill/exec/record/RecordIterator.java   |   6 +-
 6 files changed, 330 insertions(+), 158 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/da241134/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java
index b029154..0b9aeb6 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java
@@ -23,6 +23,7 @@ import 
org.apache.drill.exec.physical.impl.aggregate.HashAggTemplate;
 import 
org.apache.drill.exec.physical.impl.broadcastsender.BroadcastSenderRootExec;
 import org.apache.drill.exec.physical.impl.flatten.FlattenRecordBatch;
 import org.apache.drill.exec.physical.impl.join.HashJoinBatch;
+import org.apache.drill.exec.physical.impl.join.MergeJoinBatch;
 import org.apache.drill.exec.physical.impl.mergereceiver.MergingRecordBatch;
 import 
org.apache.drill.exec.physical.impl.partitionsender.PartitionSenderRootExec;
 import 
org.apache.drill.exec.physical.impl.unorderedreceiver.UnorderedReceiverBatch;
@@ -51,6 +52,7 @@ public class OperatorMetricRegistry {
 register(CoreOperatorType.EXTERNAL_SORT_VALUE, 
ExternalSortBatch.Metric.class);
 register(CoreOperatorType.PARQUET_ROW_GROUP_SCAN_VALUE, 
ParquetRecordReader.Metric.class);
 register(CoreOperatorType.FLATTEN_VALUE, FlattenRecordBatch.Metric.class);
+register(CoreOperatorType.MERGE_JOIN_VALUE, MergeJoinBatch.Metric.class);
   }
 
   private static void register(final int operatorType, final Class metricDef) {

http://git-wip-us.apache.org/repos/asf/drill/blob/da241134/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index a1f783f..aea415b 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -45,7 +45,7 @@ import org.apache.drill.exec.physical.config.FlattenPOP;
 import org.apache.drill.exec.record.RecordBatchSizer;
 import org.apache.drill.exec.record.AbstractSingleRecordBatch;
 import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
-import org.apache.drill.exec.record.AbstractRecordBatchMemoryManager;
+import org.apache.drill.exec.record.RecordBatchMemoryManager;
 import org.apache.drill.exec.record.MaterializedField;
 import org.apache.drill.exec.record.RecordBatch;
 import org.apache.drill.exec.record.TransferPair;
@@ -104,11 +104,11 @@ public class FlattenRecordBatch extends 
AbstractSingleRecordBatch {
 INPUT_BATCH_COUNT,
 AVG_INPUT_BATCH_BYTES,
 AVG_INPUT_ROW_BYTES,
-TOTAL_INPUT_RECORDS,
+INPUT_RECORD_COUNT,
 OUTPUT_BATCH_COUNT,
 AVG_OUTPUT_BATCH_BYTES,
 AVG_OUTPUT_ROW_BYTES,
-TOTAL_OUTPUT_RECORDS;
+OUTPUT_RECORD_COUNT;
 
 @Override
 public int metricId() {
@@ -116,7 +116,7 @@ public class FlattenRecordBatch extends 
AbstractSingleRecordBatch {
 }
   }
 
-  private class FlattenMemoryManager extends AbstractRecordBatchMemoryManager {
+  private class FlattenMemoryManager extends RecordBatchMemoryManager {
 
 @Override
 public void update() {
@@ -152,9 +152,10 @@ public class FlattenRecordBatch extends 
AbstractSingleRecordBatch {
   // i.e. all rows fit within memory budget.

[08/11] drill git commit: DRILL-6230: Extend row set readers to handle hyper vectors

2018-04-07 Thread arina

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java
--
diff --git 
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java
 
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java
index 352ab34..c402466 100644
--- 
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java
+++ 
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java
@@ -433,7 +433,7 @@ public class TestResultSetLoaderProtocol extends 
SubOperatorTest {
 assertEquals(5, schema.size());
 assertEquals(4, schema.index("e"));
 assertEquals(4, schema.index("E"));
-rootWriter.array(4).set("e1", "e2", "e3");
+rootWriter.array(4).setObject(strArray("e1", "e2", "e3"));
 rootWriter.save();
 
 // Verify. No reason to expect problems, but might as well check.

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java
--
diff --git 
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java
 
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java
index cc2a83e..07100ed 100644
--- 
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java
+++ 
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java
@@ -29,7 +29,6 @@ import org.apache.drill.exec.record.metadata.TupleMetadata;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.accessor.ArrayReader;
 import org.apache.drill.exec.vector.accessor.ArrayWriter;
-import org.apache.drill.exec.vector.accessor.ScalarElementReader;
 import org.apache.drill.exec.vector.accessor.ScalarReader;
 import org.apache.drill.exec.vector.accessor.ScalarWriter;
 import org.apache.drill.exec.vector.accessor.TupleReader;
@@ -256,7 +255,8 @@ public class TestResultSetLoaderTorture extends 
SubOperatorTest {
 ScalarReader n2Reader;
 ScalarReader s2Reader;
 ScalarReader n3Reader;
-ScalarElementReader s3Reader;
+ArrayReader s3Array;
+ScalarReader s3Reader;
 ReadState readState;
 
 public BatchReader(TestSetup setup, RowSetReader reader, ReadState 
readState) {
@@ -272,7 +272,8 @@ public class TestResultSetLoaderTorture extends 
SubOperatorTest {
   s2Reader = m2Reader.scalar("s2");
   TupleReader m3Reader = m2Reader.tuple("m3");
   n3Reader = m3Reader.scalar("n3");
-  s3Reader = m3Reader.array("s3").elements();
+  s3Array = m3Reader.array("s3");
+  s3Reader = s3Array.scalar();
 }
 
 public void verify() {
@@ -312,7 +313,7 @@ public class TestResultSetLoaderTorture extends 
SubOperatorTest {
 
 private void verifyM2Array() {
   for (int i = 0; i < setup.m2Count; i++) {
-a2Reader.setPosn(i);
+assert(a2Reader.next());
 
 // n2: usual int
 
@@ -321,7 +322,7 @@ public class TestResultSetLoaderTorture extends 
SubOperatorTest {
 if (readState.innerCount % setup.s2Cycle == 0) {
   // Skipped values should be null
   assertTrue(
-  String.format("Row %d, entry %d", rootReader.rowIndex(), i),
+  String.format("Row %d, entry %d", rootReader.offset(), i),
   s2Reader.isNull());
 } else if (readState.innerCount % setup.s2Cycle % setup.nullCycle == 
0) {
   assertTrue(s2Reader.isNull());
@@ -338,10 +339,11 @@ public class TestResultSetLoaderTorture extends 
SubOperatorTest {
 // s3: a repeated VarChar
 
 if (readState.innerCount % setup.s3Cycle == 0) {
-  assertEquals(0, s3Reader.size());
+  assertEquals(0, s3Array.size());
 } else {
   for (int j = 0; j < setup.s3Count; j++) {
-assertEquals(setup.s3Value + (readState.innerCount * setup.s3Count 
+ j), s3Reader.getString(j));
+assertTrue(s3Array.next());
+assertEquals(setup.s3Value + (readState.innerCount * setup.s3Count 
+ j), s3Reader.getString());
   }
 }
 readState.innerCount++;

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java
--
diff --git 
a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java
 
b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java
index a511a0a..0882a1c 100644
---

[10/11] drill git commit: DRILL-6284: Add operator metrics for batch sizing for flatten

2018-04-07 Thread arina

DRILL-6284: Add operator metrics for batch sizing for flatten


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/77f5e901
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/77f5e901
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/77f5e901

Branch: refs/heads/master
Commit: 77f5e901370c4c23195ac226d140e8f3eca5f71d
Parents: 4f2182e
Author: Padma Penumarthy 
Authored: Tue Mar 20 13:44:50 2018 -0700
Committer: Arina Ielchiieva 
Committed: Fri Apr 6 12:06:49 2018 +0300

--
 .../drill/exec/ops/OperatorMetricRegistry.java  |  2 +
 .../drill/exec/physical/config/FlattenPOP.java  |  4 +-
 .../impl/flatten/FlattenRecordBatch.java| 42 
 .../AbstractRecordBatchMemoryManager.java   | 53 
 .../apache/drill/exec/proto/UserBitShared.java  | 20 ++--
 .../exec/proto/beans/CoreOperatorType.java  |  4 +-
 protocol/src/main/protobuf/UserBitShared.proto  |  1 +
 7 files changed, 118 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/77f5e901/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java
index 0424332..b029154 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java
@@ -21,6 +21,7 @@ import org.apache.drill.exec.physical.impl.ScreenCreator;
 import org.apache.drill.exec.physical.impl.SingleSenderCreator;
 import org.apache.drill.exec.physical.impl.aggregate.HashAggTemplate;
 import 
org.apache.drill.exec.physical.impl.broadcastsender.BroadcastSenderRootExec;
+import org.apache.drill.exec.physical.impl.flatten.FlattenRecordBatch;
 import org.apache.drill.exec.physical.impl.join.HashJoinBatch;
 import org.apache.drill.exec.physical.impl.mergereceiver.MergingRecordBatch;
 import 
org.apache.drill.exec.physical.impl.partitionsender.PartitionSenderRootExec;
@@ -49,6 +50,7 @@ public class OperatorMetricRegistry {
 register(CoreOperatorType.HASH_JOIN_VALUE, HashJoinBatch.Metric.class);
 register(CoreOperatorType.EXTERNAL_SORT_VALUE, 
ExternalSortBatch.Metric.class);
 register(CoreOperatorType.PARQUET_ROW_GROUP_SCAN_VALUE, 
ParquetRecordReader.Metric.class);
+register(CoreOperatorType.FLATTEN_VALUE, FlattenRecordBatch.Metric.class);
   }
 
   private static void register(final int operatorType, final Class metricDef) {

http://git-wip-us.apache.org/repos/asf/drill/blob/77f5e901/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/FlattenPOP.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/FlattenPOP.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/FlattenPOP.java
index 42e6870..f3499bf 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/FlattenPOP.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/FlattenPOP.java
@@ -27,6 +27,7 @@ import org.apache.drill.exec.physical.base.AbstractBase;
 import org.apache.drill.exec.physical.base.AbstractSingle;
 import org.apache.drill.exec.physical.base.PhysicalOperator;
 import org.apache.drill.exec.physical.base.PhysicalVisitor;
+import org.apache.drill.exec.proto.UserBitShared;
 
 import java.util.Iterator;
 import java.util.List;
@@ -67,7 +68,6 @@ public class FlattenPOP extends AbstractSingle {
 
   @Override
   public int getOperatorType() {
-// TODO - add this operator to the protobuf definition
-return 0;
+return UserBitShared.CoreOperatorType.FLATTEN_VALUE;
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/77f5e901/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index 7509809..a1f783f 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -40,6 +40,7 @@ import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.expr.ValueVectorReadExpression;
 import

[04/11] drill git commit: DRILL-6271: Updated copyright range in NOTICE

2018-04-07 Thread arina

DRILL-6271: Updated copyright range in NOTICE

closes #1188


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/67669a00
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/67669a00
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/67669a00

Branch: refs/heads/master
Commit: 67669a000c19a39510118f20366f3a189809200a
Parents: a054945
Author: dvjyothsna 
Authored: Fri Mar 23 17:13:46 2018 -0700
Committer: Arina Ielchiieva 
Committed: Fri Apr 6 11:58:42 2018 +0300

--
 NOTICE| 2 +-
 distribution/src/resources/NOTICE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/67669a00/NOTICE
--
diff --git a/NOTICE b/NOTICE
index abdcb91..21b3e34 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
 Apache Drill
-Copyright 2013-2014 The Apache Software Foundation
+Copyright 2013-2018 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

http://git-wip-us.apache.org/repos/asf/drill/blob/67669a00/distribution/src/resources/NOTICE
--
diff --git a/distribution/src/resources/NOTICE 
b/distribution/src/resources/NOTICE
index bd17b61..8a85d07 100644
--- a/distribution/src/resources/NOTICE
+++ b/distribution/src/resources/NOTICE
@@ -1,5 +1,5 @@
 Apache Drill
-Copyright 2013-2014 The Apache Software Foundation
+Copyright 2013-2018 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

[02/11] drill git commit: DRILL-6303: Provide a button to copy the Drillbit's JStack shown in /threads

2018-04-07 Thread arina

DRILL-6303: Provide a button to copy the Drillbit's JStack shown in /threads

Provides a button in the web UI to copy the thread stack to the user's 
clipboard.

closes #1199


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2e1568ce
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2e1568ce
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2e1568ce

Branch: refs/heads/master
Commit: 2e1568cecd4aafb87c092c6b403a60366cc8f107
Parents: cc440af
Author: Kunal Khatua 
Authored: Sun Mar 25 23:09:43 2018 -0700
Committer: Arina Ielchiieva 
Committed: Fri Apr 6 11:54:46 2018 +0300

--
 .../src/main/resources/rest/threads/threads.ftl | 27 ++--
 1 file changed, 25 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/2e1568ce/exec/java-exec/src/main/resources/rest/threads/threads.ftl
--
diff --git a/exec/java-exec/src/main/resources/rest/threads/threads.ftl 
b/exec/java-exec/src/main/resources/rest/threads/threads.ftl
index e393164..14d22f2 100644
--- a/exec/java-exec/src/main/resources/rest/threads/threads.ftl
+++ b/exec/java-exec/src/main/resources/rest/threads/threads.ftl
@@ -17,18 +17,41 @@
   back
   
   
+  Auto Refresh 
+ 
+  
+  
+
+  
   
   
   
   
+var statusUrl = location.protocol + "//" + location.host + 
"/status/threads";
+var refreshInterval = 3;
+document.getElementById('refreshTimeLabel').innerHTML = refreshInterval+" 
sec"; 
 var update = function() {
-  $.get("/status/threads", function(data) {
+  $.get(statusUrl, function(data) {
 $("#mainDiv").html("" + data + "");
   });
 };
 
+//Ref: https://stackoverflow.com/a/36640126/8323038
+function copyThreads() {
+  if (document.selection) { 
+var range = document.body.createTextRange();
+range.moveToElementText(document.getElementById("mainDiv"));
+range.select().createTextRange();
+document.execCommand("copy");
+  } else if (window.getSelection) {
+var range = document.createRange();
+range.selectNode(document.getElementById("mainDiv"));
+window.getSelection().addRange(range);
+document.execCommand("copy");
+  }
+}
 update();
-setInterval(update, 2000);
+setInterval(update, refreshInterval*1000);

[07/11] drill git commit: DRILL-6230: Extend row set readers to handle hyper vectors

2018-04-07 Thread arina

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/VectorPrinter.java
--
diff --git 
a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/VectorPrinter.java
 
b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/VectorPrinter.java
deleted file mode 100644
index 2056220..000
--- 
a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/VectorPrinter.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.test.rowSet.test;
-
-import org.apache.drill.exec.vector.UInt4Vector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.VarCharVector;
-
-import com.google.common.base.Charsets;
-
-/**
- * Handy tool to visualize string and offset vectors for
- * debugging.
- */
-
-public class VectorPrinter {
-
-  public static void printOffsets(UInt4Vector vector, int start, int length) {
-header(vector, start, length);
-for (int i = start, j = 0; j < length; i++, j++) {
-  if (j > 0) {
-System.out.print(" ");
-  }
-  System.out.print(vector.getAccessor().get(i));
-}
-System.out.print("], addr = ");
-System.out.println(vector.getBuffer().addr());
-  }
-
-  public static void printStrings(VarCharVector vector, int start, int length) 
{
-printOffsets(vector.getOffsetVector(), start, length + 1);
-header(vector, start, length);
-System.out.println();
-for (int i = start, j = 0; j < length; i++, j++) {
-  System.out.print("  ");
-  System.out.print(i);
-  System.out.print(": \"");
-  System.out.print(stringAt(vector, i));
-  System.out.println("\"");
-}
-System.out.println("]");
-  }
-
-  public static void header(ValueVector vector, int start, int length) {
-System.out.print(vector.getClass());
-System.out.print(": (");
-System.out.print(start);
-System.out.print(" - ");
-System.out.print(start + length - 1);
-System.out.print("): [");
-  }
-
-  public static String stringAt(VarCharVector vector, int i) {
-return new String(vector.getAccessor().get(i), Charsets.UTF_8);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
--
diff --git a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java 
b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
index 109500a..513f50b 100644
--- a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
+++ b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
@@ -823,4 +823,19 @@ public final class DrillBuf extends AbstractByteBuf 
implements AutoCloseable {
   historicalLog.buildHistory(sb, indent + 1, verbosity.includeStackTraces);
 }
   }
+
+  /**
+   * Convenience method to read buffer bytes into a newly allocated byte
+   * array.
+   *
+   * @param srcOffset the offset into this buffer of the data to read
+   * @param length number of bytes to read
+   * @return byte array with the requested bytes
+   */
+
+  public byte[] unsafeGetMemory(int srcOffset, int length) {
+byte buf[] = new byte[length];
+PlatformDependent.copyMemory(addr + srcOffset, buf, 0, length);
+return buf;
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/codegen/templates/ColumnAccessors.java
--
diff --git a/exec/vector/src/main/codegen/templates/ColumnAccessors.java 
b/exec/vector/src/main/codegen/templates/ColumnAccessors.java
index 14ec1e8..4836099 100644
--- a/exec/vector/src/main/codegen/templates/ColumnAccessors.java
+++ b/exec/vector/src/main/codegen/templates/ColumnAccessors.java
@@ -1,3 +1,4 @@
+<#macro copyright>
 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -16,9 +17,10 @@
  * limitations under the License.
  */
 
+// This class is generated using Freemarker and the ${.template_name} template.
+

[01/11] drill git commit: DRILL-6279: Indicate operators that spilled in-memory data to disk on Web UI

[06/11] drill git commit: DRILL-6230: Extend row set readers to handle hyper vectors

[09/11] drill git commit: DRILL-6230: Extend row set readers to handle hyper vectors

[03/11] drill git commit: DRILL-6287: apache-release profile should be disabled by default

[11/11] drill git commit: DRILL-6296: Add operator metrics for batch sizing for merge join

[08/11] drill git commit: DRILL-6230: Extend row set readers to handle hyper vectors

[10/11] drill git commit: DRILL-6284: Add operator metrics for batch sizing for flatten

[04/11] drill git commit: DRILL-6271: Updated copyright range in NOTICE

[02/11] drill git commit: DRILL-6303: Provide a button to copy the Drillbit's JStack shown in /threads

[07/11] drill git commit: DRILL-6230: Extend row set readers to handle hyper vectors

10 matches

Site Navigation

Mail list logo

Footer information