tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prak...@apache.org
Subject [2/2] tez git commit: TEZ-2817. Tez UI: update in progress counter data for the dag vertices and tasks table (sree via pramachandran)
Date Mon, 14 Sep 2015 17:31:15 GMT
TEZ-2817. Tez UI: update in progress counter data for the dag vertices and tasks table (sree
via pramachandran)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b3ad5957
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b3ad5957
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b3ad5957

Branch: refs/heads/master
Commit: b3ad59578ba801a44c8e97b39534cf5efb90c1f1
Parents: 1e58a0e
Author: Prakash Ramachandran <pramachandran@hortonworks.com>
Authored: Mon Sep 14 23:00:13 2015 +0530
Committer: Prakash Ramachandran <pramachandran@hortonworks.com>
Committed: Mon Sep 14 23:00:13 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 tez-ui/src/main/webapp/app/scripts/app.js       | 27 ++++---
 .../app/scripts/controllers/dag_controller.js   | 14 +++-
 .../webapp/app/scripts/controllers/dag_tasks.js | 29 +++-----
 .../app/scripts/controllers/dag_vertices.js     | 23 +++++-
 .../scripts/helpers/entity-array-pollster.js    | 26 ++++++-
 .../src/main/webapp/app/scripts/helpers/misc.js | 75 +++++++++++++++++++-
 .../src/main/webapp/app/scripts/models/dag.js   |  7 +-
 8 files changed, 165 insertions(+), 37 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index bb023b1..e15b08d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -176,6 +176,7 @@ Release 0.7.1: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES:
+  TEZ-2817. Tez UI: update in progress counter data for the dag vertices and tasks table
   TEZ-2813. Tez UI: add counter data for rest api calls to AM Web Services v2
   TEZ-2660. Tez UI: need to show application page even if system metrics publish is disabled.
   TEZ-2787. Tez AM should have java.io.tmpdir=./tmp to be consistent with tasks

http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/tez-ui/src/main/webapp/app/scripts/app.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/app.js b/tez-ui/src/main/webapp/app/scripts/app.js
index 13c08cb..2ad5fdd 100644
--- a/tez-ui/src/main/webapp/app/scripts/app.js
+++ b/tez-ui/src/main/webapp/app/scripts/app.js
@@ -206,16 +206,20 @@ App.ready = function () {
 
   App.VertexInfoAdapter = App.AMInfoAdapter.extend({
     namespace: App.Configs.restNamespace.aminfoV2,
-    ajax: function(url, method, hash) {
-      var options = hash.data || {};
-      url = url.replace('__app_id__', options.appId)
-        .fmt(options.dagIdx);
-      delete options['dagIdx'];
-      delete options['appId'];
-      return this._super(url, method, hash);
+    findQuery: function(store, type, query) {
+      var record = query.metadata;
+      delete query.metadata;
+      return this.ajax(
+        this.buildURL(Ember.String.pluralize(type.typeKey),
+          record.vertexID, Em.Object.create(record)), 'GET', { data: query});
     },
-    pathForType: function() {
-      return 'verticesInfo?dagID=%@';
+    buildURL: function(type, id, record) {
+      var url = this._super(type, undefined, record);
+      return url.replace('__app_id__', record.get('appID'))
+        .fmt(record.get('dagID'), id, record.get('counters'));
+    },
+    pathForType: function(typeName) {
+      return 'verticesInfo?dagID=%@&vertexID=%@&counters=%@';
     }
   });
 
@@ -231,12 +235,13 @@ App.ready = function () {
     buildURL: function(type, id, record) {
       var url = this._super(type, undefined, record);
       return url.replace('__app_id__', record.get('appID'))
-        .fmt(record.get('dagID'), id);
+        .fmt(record.get('dagID'), id, record.get('counters'));
     },
     pathForType: function(typeName) {
-      return 'tasksInfo?dagID=%@&taskID=%@';
+      return 'tasksInfo?dagID=%@&taskID=%@&counters=%@';
     }
   });
+
 };
 
 $.ajaxPrefilter(function(options, originalOptions, jqXHR) {

http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
index 8804813..cd750d0 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
@@ -80,6 +80,7 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper,
{
           ["loading", "id", "model.status"].forEach(function(item) {
             Em.addObserver(that, item, that.startAMInfoUpdateService);
           });
+          that.startAMInfoUpdateService();
         }
       }
     });
@@ -121,9 +122,12 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper,
{
 
     if (!dagId) return;
 
-    verticesInfoLoader = this.store.find('vertexInfo', {
-      appId: that.get('applicationId'),
-      dagIdx: that.get('idx')
+    verticesInfoLoader = this.store.findQuery('vertexInfo', {
+      metadata: {
+        appID: that.get('applicationId'),
+        dagID: that.get('idx'),
+        counters: App.get('vertexCounters')
+      }
     });
 
     verticesInfoLoader.then(function(verticesInfo) {
@@ -156,6 +160,10 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper,
{
         Em.addObserver(that, item, that.stopAMInfoUpdateService);
       });
     }
+    else {
+      that.updateAMDagInfo();
+      that.updateAMVerticesInfo();
+    }
   },
 
   dostopAMInfoUpdateService: function() {

http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
index 0bd9df0..567b608 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
@@ -39,8 +39,8 @@ App.DagTasksController = App.TablePageController.extend({
   },
 
   pollsterControl: function () {
-    if(this.get('controllers.dag.status') == 'RUNNING' &&
-        this.get('controllers.dag.amWebServiceVersion') != '1' &&
+    if(this.get('status') == 'RUNNING' &&
+        this.get('amWebServiceVersion') != '1' &&
         !this.get('loading') &&
         this.get('isActive') &&
         this. get('rowsDisplayed.length') > 0) {
@@ -49,34 +49,27 @@ App.DagTasksController = App.TablePageController.extend({
     else {
       this.get('pollster').stop();
     }
-  }.observes('controllers.dag.status',
-      'controllers.dag.amWebServiceVersion',
-      'rowsDisplayed',
-      'loading',
-      'isActive'),
-
-  reset: function () {
-    this.set('pollster.options', null);
-  },
+  }.observes('status', 'amWebServiceVersion', 'rowsDisplayed', 'loading', 'isActive'),
 
   pollsterOptionsObserver: function () {
     var rows = this.get('rowsDisplayed');
     this.set('pollster.targetRecords', rows);
 
-    rows = rows.filter(function (row) {
-      return row.get('status') != 'SUCCEEDED';
-    });
-
     this.set('pollster.options', (rows && rows.length) ? {
-      appID: this.get('controllers.dag.model.applicationId'),
-      dagID: this.get('controllers.dag.model.idx'),
+      appID: this.get('applicationId'),
+      dagID: this.get('idx'),
+      counters: this.get('countersDisplayed'),
       taskID: rows.map(function (row) {
           var taskIndex = App.Helpers.misc.getIndexFromId(row.get('id')),
           vertexIndex = App.Helpers.misc.getIndexFromId(row.get('vertexID'));
           return '%@_%@'.fmt(vertexIndex, taskIndex);
         }).join(',')
     } : null);
-  }.observes('controllers.dag.model.applicationId', 'controllers.dag.model.idx', 'rowsDisplayed'),
+  }.observes('applicationId', 'idx', 'rowsDisplayed', 'counters'),
+
+  countersDisplayed: function () {
+    return App.Helpers.misc.getCounterQueryParam(this.get('columns'));
+  }.property('columns'),
 
   beforeLoad: function () {
     var dagController = this.get('controllers.dag'),

http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
index ea25818..5f60d5d 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
@@ -26,6 +26,11 @@ App.DagVerticesController = App.TablePageController.extend({
 
   cacheDomain: Ember.computed.alias('controllers.dag.id'),
 
+  pollster: {},
+  amInfoUpdateServiceObserver: function () {
+    this.set('pollster.isRunning', !!this.get('amInfoUpdateService'));
+  }.observes('amInfoUpdateService').on('init'),
+
   beforeLoad: function () {
     var dagController = this.get('controllers.dag'),
         model = dagController.get('model');
@@ -69,7 +74,8 @@ App.DagVerticesController = App.TablePageController.extend({
         metadata: {
           appId: that.get('applicationId'),
           dagIdx: that.get('idx'),
-          vertexIds: runningVerticesIdx.join(',')
+          vertexIds: runningVerticesIdx.join(','),
+          counters: counters
         }
       }).then(function(vertexProgressInfo) {
         that.set('controllers.dag.amVertexInfo', vertexProgressInfo);
@@ -79,10 +85,18 @@ App.DagVerticesController = App.TablePageController.extend({
     }
   },
 
+  _onColumnChange: function () {
+    App.set('vertexCounters', App.Helpers.misc.getCounterQueryParam(this.get('columns')));
+  }.observes('columns').on('init'),
+
   overlayVertexInfo: function(vertex, amVertexInfo) {
     if (Em.isNone(amVertexInfo) || Em.isNone(vertex)) return;
     amVertexInfo.set('_amInfoLastUpdatedTime', moment());
     vertex.setProperties(amVertexInfo.getProperties('status', 'progress', '_amInfoLastUpdatedTime'));
+
+    vertex.set('counterGroups',
+      App.Helpers.misc.mergeCounterInfo(vertex.get('counterGroups'), amVertexInfo.get('counters')).slice(0)
+    );
   },
 
   updateVertexInfo: function() {
@@ -152,6 +166,13 @@ App.DagVerticesController = App.TablePageController.extend({
         }
       },
       {
+        id: 'progress',
+        headerCellName: 'Progress',
+        contentPath: 'progress',
+        observePath: true,
+        templateName: 'components/basic-table/progress-cell'
+      },
+      {
         id: 'startTime',
         headerCellName: 'Start Time',
         contentPath: 'startTime',

http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js b/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js
index 040ec8f..fb3dd76 100644
--- a/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js
+++ b/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js
@@ -64,6 +64,10 @@ App.Helpers.EntityArrayPollster = App.Helpers.Pollster.extend({
     }
   },
 
+  _optionObserver: function () {
+    Em.run.later(this, this.onPoll, 10);
+  }.observes('options'),
+
   _callIfRunning: function (that, funName) {
     return function (data) {
       var fun = that.get(funName);
@@ -93,9 +97,27 @@ App.Helpers.EntityArrayPollster = App.Helpers.Pollster.extend({
 
     if(polledRecords && targetRecords) {
       targetRecords.forEach(function (row) {
-        var info = polledRecords.findBy('id', row.get('id'));
-        if(info) {
+        var info = polledRecords.findBy('id', row.get('id')),
+             merge = !!info;
+
+        if(merge && row.get('progress') && info.get('progress')) {
+          if(row.get('progress') >= info.get('progress')) {
+            merge = false;
+          }
+        }
+
+        if(merge) {
           row.setProperties(info.getProperties.apply(info, mergeProperties));
+
+          if(info.get('counters')) {
+            row.set('counterGroups',
+              App.Helpers.misc.mergeCounterInfo(
+                row.get('counterGroups'),
+                info.get('counters')
+              ).slice(0)
+            );
+          }
+
         }
       });
     }

http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
index 223b686..ddd7d6a 100644
--- a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
+++ b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
@@ -47,6 +47,8 @@ App.Helpers.misc = {
         return 'success';
       case 'UNDEFINED':
         return 'unknown';
+      case 'SCHEDULED':
+        return 'schedule';
       default:
         return 'submitted';
     }
@@ -158,7 +160,7 @@ App.Helpers.misc = {
    * @param counterConfigs Array
    * @return Normalized configurations
    */
-  normalizeCounterConfigs: function (counterConfigs) {
+  normalizeCounterConfigs: function (counterConfigs, inProgress) {
     return counterConfigs.map(function (configuration) {
       var groupName = configuration.counterGroupName || configuration.groupId,
           counterName = configuration.counterName || configuration.counterId;
@@ -169,6 +171,12 @@ App.Helpers.misc = {
       );
       configuration.id = '%@/%@'.fmt(groupName, counterName),
 
+      configuration.observePath = true;
+      configuration.contentPath = 'counterGroups';
+      configuration.counterGroupName = groupName;
+      configuration.counterName = counterName;
+      configuration.searchAndSortable = !inProgress;
+
       configuration.getSortValue = App.Helpers.misc.getCounterCellContent;
       configuration.getCellContent =
           configuration.getSearchValue = App.Helpers.misc.getCounterCellContentFormatted;
@@ -176,6 +184,71 @@ App.Helpers.misc = {
     });
   },
 
+  getCounterQueryParam: function (columns) {
+    var counterHash = {},
+        counters = [];
+
+    columns.forEach(function (column) {
+      var groupName = column.get('counterGroupName'),
+          counterName = column.get('counterName');
+      if(column.get('contentPath') == 'counterGroups') {
+        counterHash[groupName] = counterHash[groupName] || [];
+        counterHash[groupName].push(counterName);
+      }
+    });
+    for(var groupName in counterHash) {
+      counters.push('%@/%@'.fmt(groupName, counterHash[groupName].join(',')));
+    }
+
+    return counters.join(';');
+  },
+
+  /*
+   * Merges counter information from AM counter object into ATS counters array
+   */
+  mergeCounterInfo: function (targetATSCounters, sourceAMCounters) {
+    var atsCounters, atsCounter,
+        counters;
+
+    targetATSCounters = targetATSCounters || [];
+
+    try{
+      for(var counterGroupName in sourceAMCounters) {
+        counters = sourceAMCounters[counterGroupName],
+        atsCounters = targetATSCounters.findBy('counterGroupName', counterGroupName);
+        if(!atsCounters) {
+          atsCounters = [];
+          targetATSCounters.push({
+            counterGroupName: counterGroupName,
+            counterGroupDisplayName: counterGroupName,
+            counters: atsCounters
+          });
+        }
+        else {
+          atsCounters = atsCounters.counters;
+        }
+        for(var counterName in counters) {
+          atsCounter = atsCounters.findBy('counterName', counterName);
+          if(atsCounter) {
+            Em.set(atsCounter, 'counterValue', counters[counterName]);
+          }
+          else {
+            atsCounters.push({
+              "counterName": counterName,
+              "counterDisplayName": counterName,
+              "counterValue": counters[counterName]
+            });
+          }
+        }
+      }
+    }
+    catch(e){
+      Em.Logger.info("Counter merge failed", e);
+    }
+
+    return targetATSCounters;
+  },
+
   /*
    * Creates column definitions form configuration object array
    * @param columnConfigs Array

http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/tez-ui/src/main/webapp/app/scripts/models/dag.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/models/dag.js b/tez-ui/src/main/webapp/app/scripts/models/dag.js
index c00e591..3e48cfe 100644
--- a/tez-ui/src/main/webapp/app/scripts/models/dag.js
+++ b/tez-ui/src/main/webapp/app/scripts/models/dag.js
@@ -138,6 +138,8 @@ App.Vertex = App.AbstractEntity.extend({
     return !!f && f > 0;
   }.property('failedTasks'),
 
+  progress: DS.attr('number'),
+
   /**
    * Vertex type has to be one of the types defined in 'App.VertexType'
    * @return {string}
@@ -428,12 +430,15 @@ App.VertexInfo = DS.Model.extend({
   killedTaskAttempts: DS.attr('number'),
   pendingTasks: function() {
     return this.get('totalTasks') - this.get('runningTasks') - this.get('succeededTasks');
-  }.property('totalTasks', 'runningTasks', 'succeededTasks')
+  }.property('totalTasks', 'runningTasks', 'succeededTasks'),
+
+  counters: DS.attr('object')
 });
 
 App.TaskInfo = DS.Model.extend({
   progress: DS.attr('number'),
   status: DS.attr('string'),
+  counters: DS.attr('object')
 });
 
 App.KVDatum = DS.Model.extend({


Mime
View raw message