tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject tez git commit: TEZ-2159. Tez UI: download timeline data for offline use. (Prakash Ramachandran via hitesh)
Date Tue, 07 Apr 2015 18:48:51 GMT
Repository: tez
Updated Branches:
  refs/heads/master f2d560cbe -> 62a348cea


TEZ-2159. Tez UI: download timeline data for offline use. (Prakash Ramachandran via hitesh)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/62a348ce
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/62a348ce
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/62a348ce

Branch: refs/heads/master
Commit: 62a348ceae36939b0c5818ee626e28a5a0ae70e3
Parents: f2d560c
Author: Hitesh Shah <hitesh@apache.org>
Authored: Tue Apr 7 11:48:14 2015 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Tue Apr 7 11:48:14 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |   2 +-
 tez-ui/src/main/resources/META-INF/LICENSE.txt  |   2 +
 tez-ui/src/main/webapp/Gruntfile.js             |  22 ++
 tez-ui/src/main/webapp/app/index.html           |   2 +
 .../scripts/controllers/dag_index_controller.js |  37 +++
 .../src/main/webapp/app/scripts/helpers/io.js   | 262 +++++++++++++++++++
 .../src/main/webapp/app/scripts/helpers/misc.js | 123 +++++++++
 .../src/main/webapp/app/templates/dag/index.hbs |   5 +
 tez-ui/src/main/webapp/bower.json               |   4 +-
 9 files changed, 457 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 11b843d..0eb02a7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -8,8 +8,8 @@ INCOMPATIBLE CHANGES
   TEZ-2176. Move all logging to slf4j. (commons-logging jar no longer part of Tez tar)
   TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
 
-
 ALL CHANGES:
+  TEZ-2159. Tez UI: download timeline data for offline use.
   TEZ-2269. DAGAppMaster becomes unresponsive (post TEZ-2149).
   TEZ-2243. documentation should explicitly specify protobuf 2.5.0.
   TEZ-2232. Allow setParallelism to be called multiple times before tasks get

http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/resources/META-INF/LICENSE.txt
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/resources/META-INF/LICENSE.txt b/tez-ui/src/main/resources/META-INF/LICENSE.txt
index 6f90d29..b7c8a86 100644
--- a/tez-ui/src/main/resources/META-INF/LICENSE.txt
+++ b/tez-ui/src/main/resources/META-INF/LICENSE.txt
@@ -226,6 +226,7 @@ The Apache TEZ tez-ui bundles the following files under the MIT License:
  - jquery-mousewheel v3.1.12 (https://github.com/jquery/jquery-mousewheel) - Copyright 2006,
2014 jQuery Foundation and other contributors, https://jquery.org/
  - jquery-ui v1.11 (http://jqueryui.com/) - Copyright 2014 jQuery Foundation and other contributors
  - moment v2.8.4 (http://momentjs.com/) - authors : Tim Wood, Iskren Chernev, Moment.js contributors
+ - FileSaver.js master branch #24b303f49213b905ec9062b708f7cd43d56a5dde (https://github.com/eligrey/FileSaver.js)
authors : Eli Grey.
 
 All rights reserved.
 
@@ -256,6 +257,7 @@ The Apache Tez tez-ui bundles the following files under BSD licenses:
 (3-clause BSD license)
  - D3 v3.4.11 (http://d3js.org/) - Copyright (c) 2010-2014, Michael Bostock
  - ember-table v0.2.2 (https://github.com/Addepar/ember-table) - Copyright © 2012 Addepar,
Inc. All Rights Reserved.
+ - zip.js master branch #bfd76c66293305faaf9fcbb65b5ff7fe2dbe621a (https://github.com/gildas-lormeau/zip.js)
 
 All rights reserved.
 

http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/webapp/Gruntfile.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/Gruntfile.js b/tez-ui/src/main/webapp/Gruntfile.js
index 95e57da..98f62b1 100644
--- a/tez-ui/src/main/webapp/Gruntfile.js
+++ b/tez-ui/src/main/webapp/Gruntfile.js
@@ -281,6 +281,17 @@ module.exports = function (grunt) {
           {
             expand: true,
             flatten: true,
+            cwd: '<%= yeoman.app %>',
+            dest: '<%= yeoman.dist %>/scripts/zip.js',
+            src: [
+              'bower_components/zip.js/WebContent/z-worker.js',
+              'bower_components/zip.js/WebContent/inflate.js',
+              'bower_components/zip.js/WebContent/deflate.js',
+            ]
+          },
+          {
+            expand: true,
+            flatten: true,
             src: '<%= yeoman.app %>/bower_components/jquery-ui/themes/smoothness/images/*',
             dest: '<%= yeoman.dist %>/styles/images/'
           },
@@ -326,6 +337,17 @@ module.exports = function (grunt) {
           },
           {
             expand: true,
+            flatten: true,
+            cwd: '<%= yeoman.app %>',
+            dest: '.tmp/scripts/zip.js',
+            src: [
+              'bower_components/zip.js/WebContent/z-worker.js',
+              'bower_components/zip.js/WebContent/inflate.js',
+              'bower_components/zip.js/WebContent/deflate.js',
+            ]
+          },
+          {
+            expand: true,
             flatten: false,
             cwd: '<%= yeoman.app %>',
             src: 'bower_components/**',

http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/webapp/app/index.html
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/index.html b/tez-ui/src/main/webapp/app/index.html
index b2d1ac3..f87b2a6 100644
--- a/tez-ui/src/main/webapp/app/index.html
+++ b/tez-ui/src/main/webapp/app/index.html
@@ -60,6 +60,8 @@
     <script src="bower_components/ember-table/dist/ember-table.js"></script>
     <script src="bower_components/ember-addons.bs_for_ember/dist/js/bs-nav.min.js"></script>
     <script src="bower_components/d3/d3.js"></script>
+    <script src="bower_components/zip.js/WebContent/zip.js"></script>
+    <script src="bower_components/FileSaver.js/FileSaver.min.js"></script>
     <!-- endbuild -->
 
     <!-- build:js(.tmp) scripts/templates.js -->

http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js
index 7576149..416f500 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js
@@ -22,6 +22,43 @@ App.DagIndexController = Em.ObjectController.extend(App.ModelRefreshMixin,
{
 
   needs: 'dag',
 
+  actions: {
+    downloadDagJson: function() {
+      var dagID = this.get('id');
+      var downloader = App.Helpers.misc.downloadDAG(this.get('id'), {
+        batchSize: 500,
+        onSuccess: function() {
+          Bootstrap.ModalManager.close('downloadModal');
+        },
+        onFailure: function() {
+          $('#modalMessage').html('<i class="fa fa-lg fa-exclamation-circle margin-small-horizontal"
' +
+          'style="color:red"></i>&nbsp;Error downloading data');
+        }
+      });
+      this.set('tmpDownloader', downloader);
+      var modalDialogView = Ember.View.extend({
+        template: Em.Handlebars.compile(
+          '<p id="modalMessage"><i class="fa fa-lg fa-spinner fa-spin margin-small-horizontal"
' + 
+          'style="color:green"></i>Downloading data for dag %@</p>'.fmt(dagID)
+        )
+      });
+      var buttons = [
+        Ember.Object.create({title: 'Cancel', dismiss: 'modal', clicked: 'cancelDownload'})
+      ];
+      Bootstrap.ModalManager.open('downloadModal', 'Download data',
+        modalDialogView, buttons, this);
+    },
+
+    cancelDownload: function() {
+      var currentDownloader = this.get('tmpDownloader');
+      if (!!currentDownloader) {
+        currentDownloader.cancel();
+      }
+      this.set('tmpDownloader', undefined);
+    }
+
+  },
+
   load: function () {
     var dag = this.get('controllers.dag.model'),
         controller = this.get('controllers.dag'),

http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/webapp/app/scripts/helpers/io.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/io.js b/tez-ui/src/main/webapp/app/scripts/helpers/io.js
new file mode 100644
index 0000000..e959e0a
--- /dev/null
+++ b/tez-ui/src/main/webapp/app/scripts/helpers/io.js
@@ -0,0 +1,262 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+zip.workerScriptsPath = "scripts/zip.js/";
+
+App.Helpers.io = {
+  /* Allow queuing of downloads and then get a callback once all the downloads are done.
+   * sample usage.
+   * var downloader = App.Helpers.io.fileDownloader();
+   * downloader.queueItem({
+   *   url: 'http://....',
+   *   onItemFetched: function(data, context) {...},
+   *   context: {}, // context object gets passed back to the callback
+   * });
+   * downloader.queueItem({...}); //queue in other items
+   * downloader.finish(); // once all items are queued. items can be queued from
+   *                      // callbacks too. in that case the finish should be called
+   *                      // once all items are queued.
+   * downloader.then(successCallback).catch(failurecallback).finally(callback)
+   */
+  fileDownloader: function(options) {
+    var itemList = [],
+        opts = options || {},
+        numParallel = opts.numParallel || 5,
+        hasMoreInputs = true,
+        inProgress = 0,
+        hasFailed = false,
+        pendingRequests = {},
+        pendingRequestID = 0,
+        failureReason = 'Unknown',
+        deferredPromise = Em.RSVP.defer();
+
+    function checkForCompletion() {
+      if (hasFailed) {
+        if (inProgress == 0) {
+          deferredPromise.reject("FOOBAR");
+        }
+        return;
+      }
+
+      if (hasMoreInputs || itemList.length > 0 || inProgress > 0) {
+        return;
+      }
+
+      deferredPromise.resolve();
+    }
+
+    function getRequestId() {
+      return "req_" + pendingRequestID++;
+    }
+
+    function abortPendingRequests() {
+      $.each(pendingRequests, function(idx, val) {
+        try {
+          val.abort("abort");
+        } catch(e) {}
+      });
+    }
+
+    function markFailed(reason) {
+      if (!hasFailed) {
+        hasFailed = true;
+        failureReason = reason;
+        abortPendingRequests();
+      }
+    }
+
+    function processNext() {
+      if (inProgress >= numParallel) {
+        Em.Logger.debug("delaying download as %@ of %@ is in progress".fmt(inProgress, numParallel));
+        return;
+      }
+
+      if (itemList.length < 1) {
+        Em.Logger.debug("no items to download");
+        checkForCompletion();
+        return;
+      }
+
+      inProgress++;
+      Em.Logger.debug("starting download %@".fmt(inProgress));
+      var item = itemList.shift();
+
+      var xhr = $.getJSON(item.url);
+      var reqID = getRequestId();
+      pendingRequests[reqID] = xhr;
+
+      xhr.done(function(data, statusText, xhr) {
+        delete pendingRequests[reqID];
+
+        if ($.isFunction(item.onItemFetched)) {
+          try {
+            item.onItemFetched(data, item.context);
+          } catch (e) {
+            markFailed("invalid data");
+            inProgress--;
+            checkForCompletion();
+            return;
+          }
+        }
+
+        inProgress--;
+        processNext();
+      }).fail(function(xhr, statusText, errorObject) {
+        delete pendingRequests[reqID];
+        markFailed(statusText);
+        inProgress--;
+        checkForCompletion();
+      });
+    }
+
+    return DS.PromiseObject.create({
+      promise: deferredPromise.promise,
+
+      queueItems: function(options) {
+        options.forEach(this.queueItem);
+      },
+
+      queueItem: function(option) {
+        itemList.push(option);
+        processNext();
+      },
+
+      finish: function() {
+        hasMoreInputs = false;
+        checkForCompletion();
+      },
+
+      cancel: function() {
+        markFailed("User cancelled");
+        checkForCompletion();
+      }
+    });
+  },
+
+
+  /*
+   * allows to zip files and download that.
+   * usage: 
+   * zipHelper = App.Helpers.io.zipHelper({
+   *   onProgress: function(filename, current, total) { ...},
+   *   onAdd: function(filename) {...}
+   * });
+   * zipHelper.addFile({name: filenameinsidezip, data: data);
+   * // add all files
+   * once all files are added call the close
+   * zipHelper.close(); // or .abort to abort zip
+   * zipHelper.then(function(zippedBlob) {
+   *   saveAs(filename, zippedBlob);
+   * }).catch(failureCallback);
+   */
+  zipHelper: function(options) {
+    var opts = options || {},
+        zipFileEntry,
+        zipWriter,
+        completion = Em.RSVP.defer(),
+        fileList = [],
+        completed = 0,
+        currentIdx = -1,
+        numFiles = 0,
+        hasMoreInputs = true,
+        inProgress = false,
+        hasFailed = false;
+
+    zip.createWriter(new zip.BlobWriter(), function(writer) {
+      zipWriter = writer;
+      checkForCompletion();
+      nextFile();
+    });
+
+    function checkForCompletion() {
+      if (hasFailed) {
+        if (zipWriter) {
+          Em.Logger.debug("aborting zipping. closing file.");
+          zipWriter.close(completion.reject);
+          zipWriter = null;
+        }
+      } else {
+        if (!hasMoreInputs && numFiles == completed) {
+          Em.Logger.debug("completed zipping. closing file.");
+          zipWriter.close(completion.resolve);
+        }
+      }
+    }
+
+    function onProgress(current, total) {
+      if ($.isFunction(opts.onProgress)) {
+        opts.onProgress(fileList[currentIdx].name, current, total);
+      }
+    }
+
+    function onAdd(filename) {
+      if ($.isFunction(opts.onAdd)) {
+        opts.onAdd(filename);
+      }
+    }
+
+    function nextFile() {
+      if (hasFailed || completed == numFiles || inProgress) {
+        return;
+      }
+
+      currentIdx++;
+      var file = fileList[currentIdx];
+      inProgress = true;
+      onAdd(file.name);
+      zipWriter.add(file.name, new zip.TextReader(file.data), function() {
+        completed++;
+        inProgress = false;
+        if (currentIdx < numFiles - 1) {
+          nextFile();
+        }
+        checkForCompletion();
+      }, onProgress);
+    }
+
+    return DS.PromiseObject.create({
+      addFiles: function(files) {
+        files.forEach(this.addFile);
+      },
+
+      addFile: function(file) {
+        if (hasFailed) {
+          Em.Logger.debug("Skipping add of file %@ as zip has been aborted".fmt(file.name));
+          return;
+        }
+        numFiles++;
+        fileList.push(file);
+        if (zipWriter) {
+          Em.Logger.debug("addinng file from addFile: " + file.name);
+          nextFile();
+        }
+      },
+
+      close: function() {
+        hasMoreInputs = false;
+        checkForCompletion();
+      },
+
+      promise: completion.promise,
+
+      abort: function() {
+        hasFailed = true;
+        this.close();
+      }
+    });
+  }
+};

http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
index 608f60e..fdd69bd 100644
--- a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
+++ b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
@@ -270,6 +270,129 @@ App.Helpers.misc = {
     }
   },
 
+  downloadDAG: function(dagID, options) {
+    var opts = options || {},
+        batchSize = opts.batchSize || 1000,
+        baseurl = '%@/%@'.fmt(App.env.timelineBaseUrl, App.Configs.restNamespace.timeline),
+        itemsToDownload = [
+          {
+            url: getUrl('TEZ_DAG_ID', dagID),
+            context: { name: 'dag', type: 'TEZ_DAG_ID' },
+            onItemFetched: processSingleItem
+          },
+          {
+            url: getUrl('TEZ_VERTEX_ID', dagID),
+            context: { name: 'vertices', type: 'TEZ_VERTEX_ID', part: 0 },
+            onItemFetched: processMultipleItems
+          },
+          {
+            url: getUrl('TEZ_TASK_ID', dagID),
+            context: { name: 'tasks', type: 'TEZ_TASK_ID', part: 0 },
+            onItemFetched: processMultipleItems
+          },
+          {
+            url: getUrl('TEZ_TASK_ATTEMPT_ID', dagID),
+            context: { name: 'task_attempts', type: 'TEZ_TASK_ATTEMPT_ID', part: 0 },
+            onItemFetched: processMultipleItems
+          }
+        ],
+        numItemTypesToDownload = itemsToDownload.length,
+        downloader = App.Helpers.io.fileDownloader(),
+        zipHelper = App.Helpers.io.zipHelper({
+          onProgress: function(filename, current, total) {
+            Em.Logger.debug('%@: %@ of %@'.fmt(filename, current, total));
+          },
+          onAdd: function(filename) {
+            Em.Logger.debug('adding %@ to Zip'.fmt(filename));
+          }
+        });
+
+    function getUrl(type, dagID, fromID) {
+      var url;
+      if (type == 'TEZ_DAG_ID') {
+        url = '%@/%@/%@'.fmt(baseurl, type, dagID);
+      } else {
+        url = '%@/%@?primaryFilter=TEZ_DAG_ID:%@&limit=%@'.fmt(baseurl, type, dagID,
batchSize + 1);
+        if (!!fromID) {
+          url = '%@&fromId=%@'.fmt(url, fromID);
+        }
+      }
+      return url;
+    }
+
+    function checkIfAllDownloaded() {
+      numItemTypesToDownload--;
+      if (numItemTypesToDownload == 0) {
+        downloader.finish();
+      }
+    }
+
+    function processSingleItem(data, context) {
+      var obj = {};
+      obj[context.name] = data;
+
+      zipHelper.addFile({name: '%@.json'.fmt(context.name), data: JSON.stringify(obj, null,
2)});
+      checkIfAllDownloaded();
+    }
+
+    function processMultipleItems(data, context) {
+      var obj = {};
+      var nextBatchStart = undefined;
+
+      if (!$.isArray(data.entities)) {
+        throw "invalid data";
+      }
+
+      // need to handle no more entries , zero entries
+      if (data.entities.length > batchSize) {
+        nextBatchStart = data.entities.pop().entity;
+      }
+      obj[context.name] = data.entities;
+
+      zipHelper.addFile({name: '%@_part_%@.json'.fmt(context.name, context.part), data: JSON.stringify(obj,
null, 2)});
+
+      if (!!nextBatchStart) {
+        context.part++;
+        downloader.queueItem({
+          url: getUrl(context.type, dagID, nextBatchStart),
+          context: context,
+          onItemFetched: processMultipleItems
+        });
+      } else {
+        checkIfAllDownloaded();
+      }
+    }
+
+    downloader.queueItems(itemsToDownload);
+
+    downloader.then(function() {
+      Em.Logger.info('Finished download');
+      zipHelper.close();
+    }).catch(function() {
+      Em.Logger.error('Failed to download');
+      zipHelper.abort();
+    });
+
+    var that = this;
+    zipHelper.then(function(zippedBlob) {
+      saveAs(zippedBlob, '%@.zip'.fmt(dagID));
+      if ($.isFunction(opts.onSuccess)) {
+        opts.onSuccess();
+      }
+    }).catch(function() {
+      Em.Logger.error('zip Failed');
+      if ($.isFunction(opts.onFailure)) {
+        opts.onFailure();
+      }
+    });
+
+    return {
+      cancel: function() {
+        downloader.cancel();
+      }
+    }
+  },
+
   dagStatusUIOptions: [
     { label: 'All', id: null },
     { label: 'Submitted', id: 'SUBMITTED' },

http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/webapp/app/templates/dag/index.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/dag/index.hbs b/tez-ui/src/main/webapp/app/templates/dag/index.hbs
index c17d87f..e7c6264 100644
--- a/tez-ui/src/main/webapp/app/templates/dag/index.hbs
+++ b/tez-ui/src/main/webapp/app/templates/dag/index.hbs
@@ -33,6 +33,11 @@
       </thead>
       <tbody>
         <tr>
+          <td colspan="2">
+            {{bs-button icon="fa fa-download" title="Download data" type="info" clicked="downloadDagJson"}}
+          </td>
+        </tr>
+        <tr>
           <td>{{t 'common.applicationId'}}</td>
           <td>
             {{#if controllers.dag.enableAppIdLink}}

http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/webapp/bower.json
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/bower.json b/tez-ui/src/main/webapp/bower.json
index b3d8805..54f6f57 100644
--- a/tez-ui/src/main/webapp/bower.json
+++ b/tez-ui/src/main/webapp/bower.json
@@ -13,7 +13,9 @@
     "d3": "3.4.11",
     "ember-addons.bs_for_ember": "~0.7.0",
     "ember-table": "~0.2.4",
-    "font-awesome":"4.2.0"
+    "font-awesome":"4.2.0",
+    "FileSaver.js": "https://github.com/eligrey/FileSaver.js.git#24b303f49213b905ec9062b708f7cd43d56a5dde",
+    "zip.js": "https://github.com/gildas-lormeau/zip.js.git#bfd76c66293305faaf9fcbb65b5ff7fe2dbe621a"
   },
   "resolutions": {
     "jquery": "1.10.2",


Mime
View raw message