hive git commit: HIVE-15616 : Improve contents of qfile test output
Repository: hive
Updated Branches:
refs/heads/master bc248642a -> a01a6a348
HIVE-15616 : Improve contents of qfile test output
Signed-off-by: Ashutosh Chauhan
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a01a6a34
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a01a6a34
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a01a6a34
Branch: refs/heads/master
Commit: a01a6a348825152c8f5495017e41016fc2ce4ac4
Parents: bc24864
Author: Barna Zsombor Klara
Authored: Fri Jan 13 06:44:00 2017 -0800
Committer: Ashutosh Chauhan
Committed: Thu Apr 6 18:15:14 2017 -0700
--
.../hadoop/hive/ql/TestLocationQueries.java | 8 +-
.../control/AbstractCoreBlobstoreCliDriver.java | 11 +-
.../hive/cli/control/CoreAccumuloCliDriver.java | 10 +-
.../hadoop/hive/cli/control/CoreCliDriver.java | 12 +-
.../hive/cli/control/CoreCompareCliDriver.java | 12 +-
.../hive/cli/control/CoreHBaseCliDriver.java| 9 +-
.../cli/control/CoreHBaseNegativeCliDriver.java | 9 +-
.../hive/cli/control/CoreNegativeCliDriver.java | 12 +-
.../hive/cli/control/CorePerfCliDriver.java | 12 +-
.../hadoop/hive/ql/QTestProcessExecResult.java | 6 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java| 146 +++
.../hadoop/hive/ql/parse/CoreParseNegative.java | 19 ++-
12 files changed, 157 insertions(+), 109 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
--
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
index 0688846..c17ca10 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
@@ -53,10 +53,11 @@ public class TestLocationQueries extends BaseTestQueries {
* @return non-zero if it failed
*/
@Override
-public int checkCliDriverResults(String tname) throws Exception {
+public QTestProcessExecResult checkCliDriverResults(String tname) throws
Exception {
File logFile = new File(logDir, tname + ".out");
int failedCount = 0;
+ StringBuilder fileNames = new StringBuilder("Files failing the location
check:");
FileReader fr = new FileReader(logFile);
BufferedReader in = new BufferedReader(fr);
try {
@@ -69,19 +70,20 @@ public class TestLocationQueries extends BaseTestQueries {
File f = new File(m.group(1));
if (!f.getName().equals(locationSubdir)) {
failedCount++;
+ fileNames.append(f.getName()).append("\r\n");
}
locationCount++;
}
}
// we always have to find at least one location, otw the test is
useless
if (locationCount == 0) {
- return Integer.MAX_VALUE;
+ return QTestProcessExecResult.create(Integer.MAX_VALUE, "0 locations
tested");
}
} finally {
in.close();
}
- return failedCount;
+ return QTestProcessExecResult.create(failedCount, fileNames.toString());
}
public CheckResults(String outDir, String logDir, MiniClusterType miniMr,
http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
--
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index 9c97c31..02abe53 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -29,6 +29,7 @@ import
org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveVariableSource;
import org.apache.hadoop.hive.conf.VariableSubstitution;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
@@ -139,12 +140,14 @@ public abstract class AbstractCoreBlobstoreCliDriver
extends CliAdapter {
if ((ecode == 0) ^ expectSuccess) {
qt.failed(ecode, fname, debugHint);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
-q
svn commit: r1009947 - in /websites/staging/hive/trunk/content: ./ downloads.html javadoc.html
Author: buildbot Date: Fri Apr 7 00:04:36 2017 New Revision: 1009947 Log: Staging update by buildbot for hive Modified: websites/staging/hive/trunk/content/ (props changed) websites/staging/hive/trunk/content/downloads.html websites/staging/hive/trunk/content/javadoc.html Propchange: websites/staging/hive/trunk/content/ -- --- cms:source-revision (original) +++ cms:source-revision Fri Apr 7 00:04:36 2017 @@ -1 +1 @@ -1787232 +1790489 Modified: websites/staging/hive/trunk/content/downloads.html == --- websites/staging/hive/trunk/content/downloads.html (original) +++ websites/staging/hive/trunk/content/downloads.html Fri Apr 7 00:04:36 2017 @@ -124,6 +124,9 @@ h2:hover > .headerlink, h3:hover > .head guaranteed to be stable. For stable releases, look in the stable directory. News¶ +07 April 2017 : release 1.2.2 available¶ +This release works with Hadoop 1.x.y, 2.x.y +You can look at the complete https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12332952&styleName=Text&projectId=12310843";>JIRA change log for this release. 8 December 2016 : release 2.1.1 available¶ This release works with Hadoop 2.x.y. Hive 1.x line will continue to be maintained with Hadoop 1.x.y support. Modified: websites/staging/hive/trunk/content/javadoc.html == --- websites/staging/hive/trunk/content/javadoc.html (original) +++ websites/staging/hive/trunk/content/javadoc.html Fri Apr 7 00:04:36 2017 @@ -121,7 +121,7 @@ h2:hover > .headerlink, h3:hover > .head Hive 2.1.1 Javadocs Hive 2.0.1 Javadocs -Hive 1.2.1 Javadocs +Hive 1.2.2 Javadocs Hive 1.1.1 Javadocs Hive 1.0.1 Javadocs Hive 0.13.1 Javadocs
svn commit: r1790489 - in /hive/cms/trunk/content: downloads.mdtext javadoc.mdtext
Author: vgumashta Date: Fri Apr 7 00:04:30 2017 New Revision: 1790489 URL: http://svn.apache.org/viewvc?rev=1790489&view=rev Log: Updating site for 1.2.2 release Modified: hive/cms/trunk/content/downloads.mdtext hive/cms/trunk/content/javadoc.mdtext Modified: hive/cms/trunk/content/downloads.mdtext URL: http://svn.apache.org/viewvc/hive/cms/trunk/content/downloads.mdtext?rev=1790489&r1=1790488&r2=1790489&view=diff == --- hive/cms/trunk/content/downloads.mdtext (original) +++ hive/cms/trunk/content/downloads.mdtext Fri Apr 7 00:04:30 2017 @@ -11,6 +11,9 @@ directory. ## News +### 07 April 2017 : release 1.2.2 available +This release works with Hadoop 1.x.y, 2.x.y +You can look at the complete [JIRA change log for this release][HIVE_1_2_2_CL]. ### 8 December 2016 : release 2.1.1 available This release works with Hadoop 2.x.y. @@ -106,6 +109,7 @@ You can look at the complete [JIRA chang [HIVE_2_0_1_CL]: https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12334886&styleName=Text&projectId=12310843 [HIVE_2_0_0_CL]: https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12332641&styleName=Text&projectId=12310843 [HIVE_1_2_1_CL]: https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12332384&styleName=Text&projectId=12310843 +[HIVE_1_2_2_CL]:https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12332952&styleName=Text&projectId=12310843 [HIVE_1_1_1_CL]: https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12329557&styleName=Text&projectId=12310843 [HIVE_1_0_1_CL]: https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12329444&styleName=Text&projectId=12310843 [HIVE_1_2_0_CL]: https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12329345&styleName=Text&projectId=12310843 Modified: hive/cms/trunk/content/javadoc.mdtext URL: http://svn.apache.org/viewvc/hive/cms/trunk/content/javadoc.mdtext?rev=1790489&r1=1790488&r2=1790489&view=diff == --- hive/cms/trunk/content/javadoc.mdtext (original) +++ hive/cms/trunk/content/javadoc.mdtext Fri Apr 7 00:04:30 2017 @@ -2,7 +2,7 @@ Title: Javadoc * [Hive 2.1.1 Javadocs][r2.1.1] * [Hive 2.0.1 Javadocs][r2.0.1] - * [Hive 1.2.1 Javadocs][r1.2.1] + * [Hive 1.2.2 Javadocs][r1.2.2] * [Hive 1.1.1 Javadocs][r1.1.1] * [Hive 1.0.1 Javadocs][r1.0.1] * [Hive 0.13.1 Javadocs][r0.13.1] @@ -15,7 +15,7 @@ javadoc and sources jars for use in an I [r2.1.1]: /javadocs/r2.1.1/api/index.html [r2.0.1]: /javadocs/r2.0.1/api/index.html -[r1.2.1]: /javadocs/r1.2.1/api/index.html +[r1.2.2]: /javadocs/r1.2.2/api/index.html [r1.1.1]: /javadocs/r1.1.1/api/index.html [r1.0.1]: /javadocs/r1.0.1/api/index.html [r0.13.1]: /javadocs/r0.13.1/api/index.html
svn commit: r1009946 - /websites/production/hive/content/javadocs/r1.2.1/
Author: vgumashta Date: Thu Apr 6 23:25:28 2017 New Revision: 1009946 Log: Removing Apache Hive 1.2.1 API Docs after adding Apache Hive 1.2.2 API Docs Removed: websites/production/hive/content/javadocs/r1.2.1/
svn commit: r1009945 - /websites/production/hive/content/javadocs/r1.1.1/
Author: vgumashta Date: Thu Apr 6 23:22:21 2017 New Revision: 1009945 Log: Adding back Apache 1.1.1 API docs that were accidentaly removed. Added: websites/production/hive/content/javadocs/r1.1.1/ - copied from r1009941, websites/production/hive/content/javadocs/r1.1.1/
svn commit: r1009940 - in /websites/production/hive/content/javadocs/r1.2.2: ./ api/ api/org/ api/org/apache/ api/org/apache/hadoop/ api/org/apache/hadoop/fs/ api/org/apache/hadoop/fs/class-use/ api/o
Author: vgumashta Date: Thu Apr 6 22:41:00 2017 New Revision: 1009940 Log: Apache Hive 1.2.2 API Docs [This commit notification would consist of 2980 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.]
svn commit: r1009942 - /websites/production/hive/content/javadocs/r1.1.1/
Author: vgumashta Date: Thu Apr 6 22:45:58 2017 New Revision: 1009942 Log: Removing Apache Hive 1.2.1 API Docs after adding Apache Hive 1.2.2 API Docs Removed: websites/production/hive/content/javadocs/r1.1.1/
[hive] Git Push Summary
Repository: hive Updated Tags: refs/tags/release-1.2.2-rc0 [deleted] f0c594d90
[hive] Git Push Summary
Repository: hive Updated Tags: refs/tags/rel/release-1.2.2 [created] e6727d2ca
hive git commit: HIVE-16341: Tez Task Execution Summary has incorrect input record counts on some operators (Jason Dere, reviewed by Gopal V)
Repository: hive
Updated Branches:
refs/heads/branch-2.3 f1e0d56b2 -> ca60eb9af
HIVE-16341: Tez Task Execution Summary has incorrect input record counts on
some operators (Jason Dere, reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ca60eb9a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ca60eb9a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ca60eb9a
Branch: refs/heads/branch-2.3
Commit: ca60eb9af4f8cd3849a5ccacfb335206dd013f10
Parents: f1e0d56
Author: Jason Dere
Authored: Thu Apr 6 12:58:16 2017 -0700
Committer: Jason Dere
Committed: Thu Apr 6 13:35:33 2017 -0700
--
.../hive/ql/exec/tez/monitoring/DAGSummary.java | 55
1 file changed, 45 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/ca60eb9a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
index 1400be4..6982fb9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
@@ -11,6 +11,7 @@ import org.apache.tez.common.counters.TaskCounter;
import org.apache.tez.common.counters.TezCounter;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.client.DAGClient;
@@ -59,24 +60,58 @@ class DAGSummary implements PrintSummary {
this.hiveCounters = hiveCounters(dagClient);
}
+ private long hiveInputRecordsFromTezCounters(String vertexName, String
inputVertexName) {
+// Get the counters for the input vertex.
+Set statusOptions = new HashSet<>(1);
+statusOptions.add(StatusGetOpts.GET_COUNTERS);
+VertexStatus inputVertexStatus = vertexStatus(statusOptions,
inputVertexName);
+final TezCounters inputVertexCounters =
inputVertexStatus.getVertexCounters();
+
+// eg, group name TaskCounter_Map_7_OUTPUT_Reducer_8, counter name
OUTPUT_RECORDS
+String groupName = formattedName("TaskCounter", inputVertexName,
vertexName);
+String counterName = "OUTPUT_RECORDS";
+
+// Do not create counter if it does not exist -
+// instead fall back to default behavior for determining input records.
+TezCounter tezCounter =
inputVertexCounters.getGroup(groupName).findCounter(counterName, false);
+if (tezCounter == null) {
+ return -1;
+} else {
+ return tezCounter.getValue();
+}
+ }
+
+ private long hiveInputRecordsFromHiveCounters(String inputVertexName) {
+// The record count from these counters may not be correct if the input
vertex has
+// edges to more than one vertex, since this value counts the records
going to all
+// destination vertices.
+
+String intermediateRecordsCounterName = formattedName(
+ReduceSinkOperator.Counter.RECORDS_OUT_INTERMEDIATE.toString(),
+inputVertexName
+);
+String recordsOutCounterName =
formattedName(FileSinkOperator.Counter.RECORDS_OUT.toString(),
+inputVertexName);
+return hiveCounterValue(intermediateRecordsCounterName) +
hiveCounterValue(recordsOutCounterName);
+ }
+
private long hiveInputRecordsFromOtherVertices(String vertexName) {
List inputVerticesList =
dag.getVertex(vertexName).getInputVertices();
long result = 0;
for (Vertex inputVertex : inputVerticesList) {
- String intermediateRecordsCounterName = formattedName(
- ReduceSinkOperator.Counter.RECORDS_OUT_INTERMEDIATE.toString(),
- inputVertex.getName()
- );
- String recordsOutCounterName =
formattedName(FileSinkOperator.Counter.RECORDS_OUT.toString(),
- inputVertex.getName());
- result += (
- hiveCounterValue(intermediateRecordsCounterName)
- + hiveCounterValue(recordsOutCounterName)
- );
+ long inputVertexRecords = hiveInputRecordsFromTezCounters(vertexName,
inputVertex.getName());
+ if (inputVertexRecords < 0) {
+inputVertexRecords =
hiveInputRecordsFromHiveCounters(inputVertex.getName());
+ }
+ result += inputVertexRecords;
}
return result;
}
+ private String formattedName(String counterName, String srcVertexName,
String destVertexName) {
+return String.format("%s_", counterName) + srcVertexName.replace(" ", "_")
+ "_OUTPUT_" + destVertexName.replace(" ", "_");
+ }
+
private String formattedName(String counterName, String vertexName) {
ret
hive git commit: HIVE-16341: Tez Task Execution Summary has incorrect input record counts on some operators (Jason Dere, reviewed by Gopal V)
Repository: hive
Updated Branches:
refs/heads/branch-2 96c2a2c08 -> f3a5049f3
HIVE-16341: Tez Task Execution Summary has incorrect input record counts on
some operators (Jason Dere, reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f3a5049f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f3a5049f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f3a5049f
Branch: refs/heads/branch-2
Commit: f3a5049f33b7aafc6d2469af48edd802ce2fde38
Parents: 96c2a2c
Author: Jason Dere
Authored: Thu Apr 6 12:58:16 2017 -0700
Committer: Jason Dere
Committed: Thu Apr 6 13:03:59 2017 -0700
--
.../hive/ql/exec/tez/monitoring/DAGSummary.java | 55
1 file changed, 45 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/f3a5049f/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
index 1400be4..6982fb9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
@@ -11,6 +11,7 @@ import org.apache.tez.common.counters.TaskCounter;
import org.apache.tez.common.counters.TezCounter;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.client.DAGClient;
@@ -59,24 +60,58 @@ class DAGSummary implements PrintSummary {
this.hiveCounters = hiveCounters(dagClient);
}
+ private long hiveInputRecordsFromTezCounters(String vertexName, String
inputVertexName) {
+// Get the counters for the input vertex.
+Set statusOptions = new HashSet<>(1);
+statusOptions.add(StatusGetOpts.GET_COUNTERS);
+VertexStatus inputVertexStatus = vertexStatus(statusOptions,
inputVertexName);
+final TezCounters inputVertexCounters =
inputVertexStatus.getVertexCounters();
+
+// eg, group name TaskCounter_Map_7_OUTPUT_Reducer_8, counter name
OUTPUT_RECORDS
+String groupName = formattedName("TaskCounter", inputVertexName,
vertexName);
+String counterName = "OUTPUT_RECORDS";
+
+// Do not create counter if it does not exist -
+// instead fall back to default behavior for determining input records.
+TezCounter tezCounter =
inputVertexCounters.getGroup(groupName).findCounter(counterName, false);
+if (tezCounter == null) {
+ return -1;
+} else {
+ return tezCounter.getValue();
+}
+ }
+
+ private long hiveInputRecordsFromHiveCounters(String inputVertexName) {
+// The record count from these counters may not be correct if the input
vertex has
+// edges to more than one vertex, since this value counts the records
going to all
+// destination vertices.
+
+String intermediateRecordsCounterName = formattedName(
+ReduceSinkOperator.Counter.RECORDS_OUT_INTERMEDIATE.toString(),
+inputVertexName
+);
+String recordsOutCounterName =
formattedName(FileSinkOperator.Counter.RECORDS_OUT.toString(),
+inputVertexName);
+return hiveCounterValue(intermediateRecordsCounterName) +
hiveCounterValue(recordsOutCounterName);
+ }
+
private long hiveInputRecordsFromOtherVertices(String vertexName) {
List inputVerticesList =
dag.getVertex(vertexName).getInputVertices();
long result = 0;
for (Vertex inputVertex : inputVerticesList) {
- String intermediateRecordsCounterName = formattedName(
- ReduceSinkOperator.Counter.RECORDS_OUT_INTERMEDIATE.toString(),
- inputVertex.getName()
- );
- String recordsOutCounterName =
formattedName(FileSinkOperator.Counter.RECORDS_OUT.toString(),
- inputVertex.getName());
- result += (
- hiveCounterValue(intermediateRecordsCounterName)
- + hiveCounterValue(recordsOutCounterName)
- );
+ long inputVertexRecords = hiveInputRecordsFromTezCounters(vertexName,
inputVertex.getName());
+ if (inputVertexRecords < 0) {
+inputVertexRecords =
hiveInputRecordsFromHiveCounters(inputVertex.getName());
+ }
+ result += inputVertexRecords;
}
return result;
}
+ private String formattedName(String counterName, String srcVertexName,
String destVertexName) {
+return String.format("%s_", counterName) + srcVertexName.replace(" ", "_")
+ "_OUTPUT_" + destVertexName.replace(" ", "_");
+ }
+
private String formattedName(String counterName, String vertexName) {
return
hive git commit: HIVE-16341: Tez Task Execution Summary has incorrect input record counts on some operators (Jason Dere, reviewed by Gopal V)
Repository: hive
Updated Branches:
refs/heads/master ecabdd3e7 -> bc248642a
HIVE-16341: Tez Task Execution Summary has incorrect input record counts on
some operators (Jason Dere, reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bc248642
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bc248642
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bc248642
Branch: refs/heads/master
Commit: bc248642a89cce0c6adc456af3452f0f6ae09774
Parents: ecabdd3
Author: Jason Dere
Authored: Thu Apr 6 12:58:16 2017 -0700
Committer: Jason Dere
Committed: Thu Apr 6 12:58:16 2017 -0700
--
.../hive/ql/exec/tez/monitoring/DAGSummary.java | 55
1 file changed, 45 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/bc248642/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
index 1400be4..6982fb9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java
@@ -11,6 +11,7 @@ import org.apache.tez.common.counters.TaskCounter;
import org.apache.tez.common.counters.TezCounter;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.client.DAGClient;
@@ -59,24 +60,58 @@ class DAGSummary implements PrintSummary {
this.hiveCounters = hiveCounters(dagClient);
}
+ private long hiveInputRecordsFromTezCounters(String vertexName, String
inputVertexName) {
+// Get the counters for the input vertex.
+Set statusOptions = new HashSet<>(1);
+statusOptions.add(StatusGetOpts.GET_COUNTERS);
+VertexStatus inputVertexStatus = vertexStatus(statusOptions,
inputVertexName);
+final TezCounters inputVertexCounters =
inputVertexStatus.getVertexCounters();
+
+// eg, group name TaskCounter_Map_7_OUTPUT_Reducer_8, counter name
OUTPUT_RECORDS
+String groupName = formattedName("TaskCounter", inputVertexName,
vertexName);
+String counterName = "OUTPUT_RECORDS";
+
+// Do not create counter if it does not exist -
+// instead fall back to default behavior for determining input records.
+TezCounter tezCounter =
inputVertexCounters.getGroup(groupName).findCounter(counterName, false);
+if (tezCounter == null) {
+ return -1;
+} else {
+ return tezCounter.getValue();
+}
+ }
+
+ private long hiveInputRecordsFromHiveCounters(String inputVertexName) {
+// The record count from these counters may not be correct if the input
vertex has
+// edges to more than one vertex, since this value counts the records
going to all
+// destination vertices.
+
+String intermediateRecordsCounterName = formattedName(
+ReduceSinkOperator.Counter.RECORDS_OUT_INTERMEDIATE.toString(),
+inputVertexName
+);
+String recordsOutCounterName =
formattedName(FileSinkOperator.Counter.RECORDS_OUT.toString(),
+inputVertexName);
+return hiveCounterValue(intermediateRecordsCounterName) +
hiveCounterValue(recordsOutCounterName);
+ }
+
private long hiveInputRecordsFromOtherVertices(String vertexName) {
List inputVerticesList =
dag.getVertex(vertexName).getInputVertices();
long result = 0;
for (Vertex inputVertex : inputVerticesList) {
- String intermediateRecordsCounterName = formattedName(
- ReduceSinkOperator.Counter.RECORDS_OUT_INTERMEDIATE.toString(),
- inputVertex.getName()
- );
- String recordsOutCounterName =
formattedName(FileSinkOperator.Counter.RECORDS_OUT.toString(),
- inputVertex.getName());
- result += (
- hiveCounterValue(intermediateRecordsCounterName)
- + hiveCounterValue(recordsOutCounterName)
- );
+ long inputVertexRecords = hiveInputRecordsFromTezCounters(vertexName,
inputVertex.getName());
+ if (inputVertexRecords < 0) {
+inputVertexRecords =
hiveInputRecordsFromHiveCounters(inputVertex.getName());
+ }
+ result += inputVertexRecords;
}
return result;
}
+ private String formattedName(String counterName, String srcVertexName,
String destVertexName) {
+return String.format("%s_", counterName) + srcVertexName.replace(" ", "_")
+ "_OUTPUT_" + destVertexName.replace(" ", "_");
+ }
+
private String formattedName(String counterName, String vertexName) {
return Stri
hive git commit: HIVE-16371: Add bitmap selection strategy for druid storage handler (Slim Bouguerra, reviewed by Jesus Camacho Rodriguez)
Repository: hive
Updated Branches:
refs/heads/master db24ce722 -> ecabdd3e7
HIVE-16371: Add bitmap selection strategy for druid storage handler (Slim
Bouguerra, reviewed by Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ecabdd3e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ecabdd3e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ecabdd3e
Branch: refs/heads/master
Commit: ecabdd3e7eff59fb83b999e4f31533f91fd6391c
Parents: db24ce7
Author: Slim Bouguerra
Authored: Thu Apr 6 11:01:09 2017 +0200
Committer: Jesus Camacho Rodriguez
Committed: Thu Apr 6 11:01:09 2017 +0200
--
.../src/java/org/apache/hadoop/hive/conf/HiveConf.java | 1 +
druid-handler/pom.xml| 1 +
.../apache/hadoop/hive/druid/io/DruidOutputFormat.java | 11 ++-
.../apache/hadoop/hive/ql/io/TestDruidRecordWriter.java | 8 ++--
4 files changed, 18 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/ecabdd3e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 6de6ed6..7d4a6a0 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2001,6 +2001,7 @@ public class HiveConf extends Configuration {
HIVE_DRUID_PASSIVE_WAIT_TIME("hive.druid.passiveWaitTimeMs", 3,
"Wait time in ms default to 30 seconds."
),
+HIVE_DRUID_BITMAP_FACTORY_TYPE("hive.druid.bitmap.type", "roaring", new
PatternSet("roaring", "concise"), "Coding algorithm use to encode the bitmaps"),
// For HBase storage handler
HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", true,
"Whether writes to HBase should be forced to the write-ahead log. \n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/ecabdd3e/druid-handler/pom.xml
--
diff --git a/druid-handler/pom.xml b/druid-handler/pom.xml
index ffc9d86..6afb5b8 100644
--- a/druid-handler/pom.xml
+++ b/druid-handler/pom.xml
@@ -329,6 +329,7 @@
org.jdbi:*
net.jpountz.lz4:*
org.apache.commons:*
+ org.roaringbitmap:*
http://git-wip-us.apache.org/repos/asf/hive/blob/ecabdd3e/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
--
diff --git
a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
index 40a2022..4385dfe 100644
---
a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
+++
b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
@@ -33,6 +33,9 @@ import io.druid.granularity.QueryGranularity;
import io.druid.query.aggregation.AggregatorFactory;
import io.druid.query.aggregation.DoubleSumAggregatorFactory;
import io.druid.query.aggregation.LongSumAggregatorFactory;
+import io.druid.segment.IndexSpec;
+import io.druid.segment.data.ConciseBitmapSerdeFactory;
+import io.druid.segment.data.RoaringBitmapSerdeFactory;
import io.druid.segment.indexing.DataSchema;
import io.druid.segment.indexing.RealtimeTuningConfig;
import io.druid.segment.indexing.granularity.GranularitySpec;
@@ -200,6 +203,12 @@ public class DruidOutputFormat implements
HiveOutputFormat implements
HiveOutputFormathttp://git-wip-us.apache.org/repos/asf/hive/blob/ecabdd3e/druid-handler/src/test/org/apache/hadoop/hive/ql/io/TestDruidRecordWriter.java
--
diff --git
a/druid-handler/src/test/org/apache/hadoop/hive/ql/io/TestDruidRecordWriter.java
b/druid-handler/src/test/org/apache/hadoop/hive/ql/io/TestDruidRecordWriter.java
index 9ec82c0..d9e01fe 100644
---
a/druid-handler/src/test/org/apache/hadoop/hive/ql/io/TestDruidRecordWriter.java
+++
b/druid-handler/src/test/org/apache/hadoop/hive/ql/io/TestDruidRecordWriter.java
@@ -37,8 +37,10 @@ import io.druid.granularity.QueryGranularities;
import io.druid.query.aggregation.AggregatorFactory;
import io.druid.query.aggregation.LongSumAggregatorFactory;
import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory;
+import io.druid.segment.IndexSpec;
import io.druid.segment.QueryableIndex;
import io.druid.segment.QueryableIndexStorageAdapter;
+import io.druid.segment.data.RoaringBitmapSerdeFactory;
import io.druid.segment
