Repository: ambari Updated Branches: refs/heads/trunk 9432be826 -> 642ba0e93
AMBARI-5078. Job Tez DAG should show spilled records per vertex. (xiwang) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/642ba0e9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/642ba0e9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/642ba0e9 Branch: refs/heads/trunk Commit: 642ba0e93fdd25abc7f018283d20119af85551b4 Parents: 9432be8 Author: Xi Wang <[email protected]> Authored: Thu Mar 13 16:53:41 2014 -0700 Committer: Xi Wang <[email protected]> Committed: Fri Mar 14 11:16:29 2014 -0700 ---------------------------------------------------------------------- ambari-web/app/messages.js | 2 ++ ambari-web/app/models/jobs/tez_dag.js | 5 +++++ ambari-web/app/templates/main/jobs/hive_job_details.hbs | 4 ++++ ambari-web/app/utils/jobs.js | 5 +++++ .../app/views/main/jobs/hive_job_details_tez_dag_view.js | 11 ++++++++--- ambari-web/app/views/main/jobs/hive_job_details_view.js | 5 ++++- 6 files changed, 28 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/642ba0e9/ambari-web/app/messages.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js index f727271..02c2af3 100644 --- a/ambari-web/app/messages.js +++ b/ambari-web/app/messages.js @@ -1954,6 +1954,7 @@ Em.I18n.translations = { 'jobs.hive.tez.tasks':'Tez Tasks', 'jobs.hive.tez.hdfs':'HDFS', 'jobs.hive.tez.localFiles':'Local Files', + 'jobs.hive.tez.spilledRecords':'Spilled Records', 'jobs.hive.tez.records':'Records', 'jobs.hive.tez.reads':'{0} reads', 'jobs.hive.tez.writes':'{0} writes', @@ -1971,6 +1972,7 @@ Em.I18n.translations = { 'jobs.hive.tez.metric.recordsRead':'Records Read', 'jobs.hive.tez.metric.recordsWrite':'Records Written', 'jobs.hive.tez.metric.tezTasks':'Tez Tasks', + 'jobs.hive.tez.metric.spilledRecords':'Spilled Records', 'jobs.hive.tez.edge.':'Unknown', 'jobs.hive.tez.edge.contains':'Contains', 'jobs.hive.tez.edge.broadcast':'Broadcast', http://git-wip-us.apache.org/repos/asf/ambari/blob/642ba0e9/ambari-web/app/models/jobs/tez_dag.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/models/jobs/tez_dag.js b/ambari-web/app/models/jobs/tez_dag.js index 38ad700..bae96ab 100644 --- a/ambari-web/app/models/jobs/tez_dag.js +++ b/ambari-web/app/models/jobs/tez_dag.js @@ -119,6 +119,11 @@ App.TezDagVertex = DS.Model.extend({ fileWriteOps : DS.attr('number'), /** + * Spilled records + */ + spilledRecords : DS.attr('number'), + + /** * HDFS usage metrics for this vertex */ hdfsReadBytes : DS.attr('number'), http://git-wip-us.apache.org/repos/asf/ambari/blob/642ba0e9/ambari-web/app/templates/main/jobs/hive_job_details.hbs ---------------------------------------------------------------------- diff --git a/ambari-web/app/templates/main/jobs/hive_job_details.hbs b/ambari-web/app/templates/main/jobs/hive_job_details.hbs index e9760ae..77ad505 100644 --- a/ambari-web/app/templates/main/jobs/hive_job_details.hbs +++ b/ambari-web/app/templates/main/jobs/hive_job_details.hbs @@ -161,6 +161,10 @@ <td>{{view.selectedVertexIODisplay.file.read.ops}} / {{view.selectedVertexIODisplay.file.read.bytes}}</td> <td>{{view.selectedVertexIODisplay.file.write.ops}} / {{view.selectedVertexIODisplay.file.write.bytes}}</td> </tr> + <tr> + <td>{{t jobs.hive.tez.spilledRecords}}</td> + <td>{{view.selectedVertex.spilledRecords}}</td> + </tr> {{#if view.selectedVertexIODisplay.records.read}} <tr> <td>{{t jobs.hive.tez.records}}</td> http://git-wip-us.apache.org/repos/asf/ambari/blob/642ba0e9/ambari-web/app/utils/jobs.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/jobs.js b/ambari-web/app/utils/jobs.js index 4dbcdbc..bbfce09 100644 --- a/ambari-web/app/utils/jobs.js +++ b/ambari-web/app/utils/jobs.js @@ -197,6 +197,11 @@ module.exports = { 'HDFS_WRITE_OPS' : 'hdfsWriteOps' }; break; + case 'org.apache.tez.common.counters.TaskCounter': + cNameToPropetyMap = { + 'SPILLED_RECORDS' : 'spilledRecords' + }; + break; case 'HIVE': cNameToPropetyMap = { 'RECORDS_READ' : 'recordReadCount', http://git-wip-us.apache.org/repos/asf/ambari/blob/642ba0e9/ambari-web/app/views/main/jobs/hive_job_details_tez_dag_view.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/views/main/jobs/hive_job_details_tez_dag_view.js b/ambari-web/app/views/main/jobs/hive_job_details_tez_dag_view.js index f470e1e..b8671d7 100644 --- a/ambari-web/app/views/main/jobs/hive_job_details_tez_dag_view.js +++ b/ambari-web/app/views/main/jobs/hive_job_details_tez_dag_view.js @@ -212,14 +212,16 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({ output : Number.MAX_VALUE, recordsRead : Number.MAX_VALUE, recordsWrite : Number.MAX_VALUE, - tezTasks : Number.MAX_VALUE + tezTasks : Number.MAX_VALUE, + spilledRecords : Number.MAX_VALUE }; dagVisualModel.maxMetrics = { input : 0, output : 0, recordsRead : 0, recordsWrite : 0, - tezTasks : 0 + tezTasks : 0, + spilledRecords : 0 }; if (dagVisualModel.nodes) { dagVisualModel.nodes.forEach(function(node) { @@ -230,6 +232,7 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({ node.metrics['recordsRead'] = vertex.get('recordReadCount'); node.metrics['recordsWrite'] = vertex.get('recordWriteCount'); node.metrics['tezTasks'] = vertex.get('tasksCount'); + node.metrics['spilledRecords'] = vertex.get('spilledRecords'); node.state = vertex.get('state'); // Min metrics dagVisualModel.minMetrics.input = Math.min(dagVisualModel.minMetrics.input, node.metrics.input); @@ -237,12 +240,14 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({ dagVisualModel.minMetrics.recordsRead = Math.min(dagVisualModel.minMetrics.recordsRead, node.metrics.recordsRead); dagVisualModel.minMetrics.recordsWrite = Math.min(dagVisualModel.minMetrics.recordsWrite, node.metrics.recordsWrite); dagVisualModel.minMetrics.tezTasks = Math.min(dagVisualModel.minMetrics.tezTasks, node.metrics.tezTasks); + dagVisualModel.minMetrics.spilledRecords = Math.min(dagVisualModel.minMetrics.spilledRecords, node.metrics.spilledRecords); // Max metrics dagVisualModel.maxMetrics.input = Math.max(dagVisualModel.maxMetrics.input, node.metrics.input); dagVisualModel.maxMetrics.output = Math.max(dagVisualModel.maxMetrics.output, node.metrics.output); dagVisualModel.maxMetrics.recordsRead = Math.max(dagVisualModel.maxMetrics.recordsRead, node.metrics.recordsRead); dagVisualModel.maxMetrics.recordsWrite = Math.max(dagVisualModel.maxMetrics.recordsWrite, node.metrics.recordsWrite); dagVisualModel.maxMetrics.tezTasks = Math.max(dagVisualModel.maxMetrics.tezTasks, node.metrics.tezTasks); + dagVisualModel.maxMetrics.spilledRecords = Math.max(dagVisualModel.maxMetrics.spilledRecords, node.metrics.spilledRecords); } }); } @@ -250,7 +255,7 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({ }.observes('[email protected]', '[email protected]', '[email protected]', '[email protected]', '[email protected]', '[email protected]', - '[email protected]'), + '[email protected]', '[email protected]'), /** * Determines layout and creates Tez graph. In the process it populates the http://git-wip-us.apache.org/repos/asf/ambari/blob/642ba0e9/ambari-web/app/views/main/jobs/hive_job_details_view.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/views/main/jobs/hive_job_details_view.js b/ambari-web/app/views/main/jobs/hive_job_details_view.js index d3a51c6..42dd238 100644 --- a/ambari-web/app/views/main/jobs/hive_job_details_view.js +++ b/ambari-web/app/views/main/jobs/hive_job_details_view.js @@ -51,7 +51,7 @@ App.MainHiveJobDetailsView = Em.View.extend({ summaryMetricType: 'input', summaryMetricTypesDisplay : [ Em.I18n.t('jobs.hive.tez.metric.input'), Em.I18n.t('jobs.hive.tez.metric.output'), Em.I18n.t('jobs.hive.tez.metric.recordsRead'), - Em.I18n.t('jobs.hive.tez.metric.recordsWrite'), Em.I18n.t('jobs.hive.tez.metric.tezTasks') ], + Em.I18n.t('jobs.hive.tez.metric.recordsWrite'), Em.I18n.t('jobs.hive.tez.metric.tezTasks'), Em.I18n.t('jobs.hive.tez.metric.spilledRecords') ], summaryMetricTypeDisplay: function(){ return Em.I18n.t('jobs.hive.tez.metric.'+this.get('summaryMetricType')); }.property('summaryMetricType'), @@ -121,6 +121,9 @@ App.MainHiveJobDetailsView = Em.View.extend({ case Em.I18n.t('jobs.hive.tez.metric.tezTasks'): summaryType = 'tezTasks'; break; + case Em.I18n.t('jobs.hive.tez.metric.spilledRecords'): + summaryType = 'spilledRecords'; + break; default: break; }
