Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id BB42C200CE1 for ; Fri, 28 Jul 2017 01:21:47 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id B947B16BD34; Thu, 27 Jul 2017 23:21:47 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id BF15216BD32 for ; Fri, 28 Jul 2017 01:21:45 +0200 (CEST) Received: (qmail 44612 invoked by uid 500); 27 Jul 2017 23:21:45 -0000 Mailing-List: contact commits-help@gobblin.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@gobblin.incubator.apache.org Delivered-To: mailing list commits@gobblin.incubator.apache.org Received: (qmail 44603 invoked by uid 99); 27 Jul 2017 23:21:44 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 27 Jul 2017 23:21:44 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D00E8E0A38; Thu, 27 Jul 2017 23:21:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hutran@apache.org To: commits@gobblin.apache.org Message-Id: <82f09ef8482b401c82b0c6c5a0d13b82@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: incubator-gobblin git commit: [GOBBLIN-9] Improve AdminUI and RestService with better sorting, filtering, auto-updates, etc. Date: Thu, 27 Jul 2017 23:21:44 +0000 (UTC) archived-at: Thu, 27 Jul 2017 23:21:47 -0000 Repository: incubator-gobblin Updated Branches: refs/heads/master 30921bf5c -> b12c35385 [GOBBLIN-9] Improve AdminUI and RestService with better sorting, filtering, auto-updates, etc. Closes #1968 from kadaan/AdminUI_Improvements Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/b12c3538 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/b12c3538 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/b12c3538 Branch: refs/heads/master Commit: b12c3538585f42a957b0712d34a8065fa3e22f20 Parents: 30921bf Author: Joel Baranick Authored: Thu Jul 27 16:21:00 2017 -0700 Committer: Hung Tran Committed: Thu Jul 27 16:21:00 2017 -0700 ---------------------------------------------------------------------- .../main/java/gobblin/admin/AdminWebServer.java | 17 +- .../src/main/resources/static/css/gobblin.css | 36 +++-- .../resources/static/css/tablesorter.theme.css | 70 ++++++++ .../src/main/resources/static/index.html | 115 +++++++++++-- .../static/js/collections/job-executions.js | 6 + .../src/main/resources/static/js/gobblin.js | 31 +++- .../src/main/resources/static/js/router.js | 6 +- .../static/js/views/job-execution-view.js | 160 +++++++++++++------ .../main/resources/static/js/views/job-view.js | 148 ++++++++++++----- .../static/js/views/key-value-table-view.js | 76 +++++++++ .../main/resources/static/js/views/over-view.js | 50 ++++-- .../resources/static/js/views/table-view.js | 94 ++++++++--- .../configuration/ConfigurationKeys.java | 4 + .../database/DatabaseJobHistoryStoreV101.java | 26 ++- .../pegasus/gobblin/rest/JobExecutionQuery.pdsc | 7 + .../gobblin.rest.jobExecutions.snapshot.json | 12 +- 16 files changed, 693 insertions(+), 165 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-admin/src/main/java/gobblin/admin/AdminWebServer.java ---------------------------------------------------------------------- diff --git a/gobblin-admin/src/main/java/gobblin/admin/AdminWebServer.java b/gobblin-admin/src/main/java/gobblin/admin/AdminWebServer.java index d037591..5c3a593 100644 --- a/gobblin-admin/src/main/java/gobblin/admin/AdminWebServer.java +++ b/gobblin-admin/src/main/java/gobblin/admin/AdminWebServer.java @@ -45,6 +45,8 @@ public class AdminWebServer extends AbstractIdleService { private final URI restServerUri; private final URI serverUri; + private final String hideJobsWithoutTasksByDefault; + private final long refreshInterval; protected Server server; public AdminWebServer(Properties properties, URI restServerUri) { @@ -54,6 +56,10 @@ public class AdminWebServer extends AbstractIdleService { this.restServerUri = restServerUri; int port = getPort(properties); this.serverUri = URI.create(String.format("http://%s:%d", getHost(properties), port)); + this.hideJobsWithoutTasksByDefault = properties.getProperty( + ConfigurationKeys.ADMIN_SERVER_HIDE_JOBS_WITHOUT_TASKS_BY_DEFAULT_KEY, + ConfigurationKeys.DEFAULT_ADMIN_SERVER_HIDE_JOBS_WITHOUT_TASKS_BY_DEFAULT); + this.refreshInterval = getRefreshInterval(properties); } @Override @@ -72,7 +78,7 @@ public class AdminWebServer extends AbstractIdleService { } private Handler buildSettingsHandler() { - final String responseTemplate = "var Gobblin = window.Gobblin || {};" + "Gobblin.settings = {restServerUrl:\"%s\"}"; + final String responseTemplate = "var Gobblin = window.Gobblin || {};" + "Gobblin.settings = {restServerUrl:\"%s\", hideJobsWithoutTasksByDefault:%s, refreshInterval:%s}"; return new AbstractHandler() { @Override @@ -81,7 +87,8 @@ public class AdminWebServer extends AbstractIdleService { if (request.getRequestURI().equals("/js/settings.js")) { response.setContentType("application/javascript"); response.setStatus(HttpServletResponse.SC_OK); - response.getWriter().println(String.format(responseTemplate, AdminWebServer.this.restServerUri.toString())); + response.getWriter().println(String.format(responseTemplate, AdminWebServer.this.restServerUri.toString(), + AdminWebServer.this.hideJobsWithoutTasksByDefault, AdminWebServer.this.refreshInterval)); baseRequest.setHandled(true); } } @@ -114,4 +121,10 @@ public class AdminWebServer extends AbstractIdleService { private static String getHost(Properties properties) { return properties.getProperty(ConfigurationKeys.ADMIN_SERVER_HOST_KEY, ConfigurationKeys.DEFAULT_ADMIN_SERVER_HOST); } + + private static long getRefreshInterval(Properties properties) { + return Long.parseLong( + properties.getProperty(ConfigurationKeys.ADMIN_SERVER_REFRESH_INTERVAL_KEY, + "" + ConfigurationKeys.DEFAULT_ADMIN_SERVER_REFRESH_INTERVAL)); + } } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-admin/src/main/resources/static/css/gobblin.css ---------------------------------------------------------------------- diff --git a/gobblin-admin/src/main/resources/static/css/gobblin.css b/gobblin-admin/src/main/resources/static/css/gobblin.css index cf0214c..4dbe675 100644 --- a/gobblin-admin/src/main/resources/static/css/gobblin.css +++ b/gobblin-admin/src/main/resources/static/css/gobblin.css @@ -90,20 +90,36 @@ Not generated by the bootstrap theme - needs to be updated if colors are changed padding: 0 10px; } -.key-value-table { - font-size: 1.2em; - line-height: 2em; +.key-value-table td:nth-child(1) { + vertical-align: top; + font-weight: bold; } -.key-value-table.key-value-centered { - width: 100%; +.key-value-table td:nth-child(2) { + border-left: 2em solid transparent; + vertical-align: top; + word-break: break-all; } -.key-value-table.key-value-centered td:nth-child(1) { - width: 50%; + +.key-value-table td div { + max-height: 100px; + overflow: scroll; } -.key-value-table td:nth-child(1) { + +table#jobs-table td { + white-space: nowrap; +} + +.summary-table { + font-size: 1.2em; + line-height: 2em; +} + +.summary-table td:nth-child(1) { text-align: right; font-weight: bold; } -.key-value-table td:nth-child(2) { + +.summary-table td:nth-child(2) { border-left: 2em solid transparent; -} + word-break: break-all; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-admin/src/main/resources/static/css/tablesorter.theme.css ---------------------------------------------------------------------- diff --git a/gobblin-admin/src/main/resources/static/css/tablesorter.theme.css b/gobblin-admin/src/main/resources/static/css/tablesorter.theme.css index 52f8f06..d725b95 100644 --- a/gobblin-admin/src/main/resources/static/css/tablesorter.theme.css +++ b/gobblin-admin/src/main/resources/static/css/tablesorter.theme.css @@ -87,6 +87,7 @@ width: 98%; margin: 0; padding: 4px 6px; + border: 1px solid #bbb; color: #333; -webkit-box-sizing: border-box; -moz-box-sizing: border-box; @@ -159,3 +160,72 @@ cursor: pointer; background-color: #e6bf99; } + +/* pager wrapper, div */ +.tablesorter-pager { + padding: 5px; +} +/* pager wrapper, in thead/tfoot */ +td.tablesorter-pager { + background-color: #e6eeee; + margin: 0; /* needed for bootstrap .pager gets a 18px bottom margin */ +} +/* pager navigation arrows */ +.tablesorter-pager img { + vertical-align: middle; + margin-right: 2px; + cursor: pointer; +} + +/* pager output text */ +.tablesorter-pager .pagedisplay { + padding: 0 5px 0 5px; + width: 50px; + text-align: center; +} + +/* pager element reset (needed for bootstrap) */ +.tablesorter-pager select { + margin: 0; + padding: 0; +} + +/*** css used when "updateArrows" option is true ***/ +/* the pager itself gets a disabled class when the number of rows is less than the size */ +.tablesorter-pager.disabled { + display: none; +} +/* hide or fade out pager arrows when the first or last row is visible */ +.tablesorter-pager .disabled { + opacity: 0.5; + filter: alpha(opacity=50); + cursor: default; +} + +div.first { + width:16px; + height:16px; + display: inline-block; + background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAACSUlEQVR4AWSSA4xcARCGG6uxL7bdixvWbsOLa9uN+nJx3a7Otm1jVa5t276dvpm07/SSfzn/NzwAALt0VAgVrCpZnWJVRaLP9FvF3vidxoOsDr0Zit8d3HTyJb+NG0q1NqxQacLrP3QbvStW/su+yF2MwVgOsMN8uHvNwyi0Zr0vFIdkpgDFrTIpkc6D2x8B8Q+VvmnWzGAsejgAUtGs0lmS2VwR8MnkSxBJFiDMKpUr0W/JTB7WZX+S9VMGBj0EwL6wbMz83+yO5MATzYEvmkfR5yGJn4PMLon1jzp8d9GLgMrBDQcfy8YHM16qXoEbQvUu8zW+CpyhLMXorF7g9Un46EXAKfEvwwb2jKWGEghYJgBWMsCazzFyAtiDWYinixCIZqC5Z3wDvQio+qNUh3FYaA7G83CJWYbrAhXwp+z4GUUAqz9DVeSLW9DR1RNGLwF+K1QIQDPp4ttFMgimHbsAZl/mXxUFaG5t5wCnVuWajWgyB4lMEfwxBCyQwRbIYFUcwORN02aMrijw6ru4Fiq7Fk18uzsE5TLg4Cj4Co8yorAq+s3gTkOhVIYlqQYY0SQ3xIqn3cG7i2syfTiexQBufUZPGoVG/E4JTGz2r7Vt+qsiI62ROyTesIIZm1tL+iIpCoymitgvtoGrJbDRFYEv9d3J17wpOqR9p/y+S8a09Y3pJX9MYPfFaSbYs8YWhuk1BbznNelffp3Yd8ocBKm3Gmx3q+um+R+FrRs8gejv3k6YNPVtx8Q5RysnrpuaMPUGRmaiODsDAKw/CRKUFv+sAAAAAElFTkSuQmCC); +} + +div.prev { + width:16px; + height:16px; + display: inline-block; + background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAACXklEQVR4AWyTA4ycARCFG6uxL7Z9FzesbUe1bZ9t27Zt27b5r63X/xXnSd5yvjfY2QMAduhIMKxE2Yg6KeoGxdf8jN/tzt8OHhRl/SVb/jSraT6wpWe8sX9wWOgbGBIaOkca02qnAz+mS54yh7n/ue3woZT6Jdu+4cnRlQ05lBoDjGYLKIVaj8VVCZo7B0ajyyZtmUtmu4E14YGRKaVWZwRDozdBojRAEKXSmcBQavRoaOtVRhSP2ZIBDTgX22bl//C6XI/5DS1WpHoKS1Idplc1myZl1c2jrxJXnpKlgU1W41wg294O3/Do3AFfd2/n52CMTC8jIL0lkCwNTjZ3jzVyZrY6u67FJYd6GmBRosPkigZnvxbQgN9BrjZiTapBTGpBI1ka3OjtHxS4rFWZHhfta0TVggArn/mSL6oA19zawPfsQm80IzE5VSD7x6Cnb4AG2FAYcNOrFxd+VYIGNLzi1IjzP0px1aWF3fzrwoCYuIRNg5N17UONUqUOCo0RC4I4v2ePOEI3ZtY0BGgmqhMTy2rwlxlfkCIgInlzBJvkqonA2cUNWCwAAbZ526cPrEix9VvevRhbVMNgsqC6dQi2IUWbS7R6m7L+tKq+bVSQa8EgxOqsOL6kBkE+s8DEghR+YfGjd0PGn5LdPKSAnD7b/PJ65YpEBSZKVUawGxrxmFh5fEEC34gU5eeAYlsye07ZI7nNNj49f7SldwKzK3JwJ5x5aEZASX0fPAKiRz/6FW6e8r5/pkeRM0/tw0sCvYLjGgOCQgQfv0DBxTei8bNv1u9NTZh6AyMzUZydAcyECTzxVTE0AAAAAElFTkSuQmCC); +} + +div.next { + width:16px; + height:16px; + display: inline-block; + background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAACWUlEQVR4AWxTA6xbYRhdrMV+se334oVzNCOabdu2qtm2zXLeatvt7V3N17N7hjve5JTf0Y8eAP5AHzU6JHRJGChhNMHP/I3//T3/O7GnhM7l1/MzrhkiStMnj95icwhmq13QvXfqL70MKJdczs3gDGd/E5DJvS5o4+vMDp8rmc2jWGmg2d0GUSjXEUvlYHxvdR1/5FvHWXJkAaqSbHX6i9VaE3wq9RZyxQYECaVaiz9JonXo3nwuHrnvpkgnKMBejE3nn2RvoozUlzqS4ne4Y2XEcjVZ5NFzo2vu2eQMcinQdU0fVjI2n0y+jmGbtCSRjLhYw5ANL2GLFBHJVsHHGUhAcdmkJJcCA40f3Xp2ZlSShm7USngJZ6yEqFCVBF6AsIaLyJebSIsVnLh4R08uBUZ/ttgELla20AAF6EiM2GqCJ17GkHUvJDzD8M0G8Hu92Y2z5y8K5H4T+GS2UgCMz64kD99iAF0pOnjtM9aS3p/yu5SigROnzsgCA1+9tevFYg2FShPsSTJdORxIVWQyDbgznqgIxZHzcoWu88+8ylAsi3YbCGWqYFySfckKiMFrnoK7woVttNp4/tqOdZp78iJ2LLiQmfFM+8Yl5KvyTpDI7WRnkrmgNPBGRew/dNo1QeOZQa58kBQ3zOtuP9YWk7nvg2KpyTQIpis8THSGJ5rDviMXissU9+WD9MdR3nn+zbrTl2+7TJ+9CCXzXBN2hj0o4IHWjJ2K464l++/KR/m/l2nq0eCMDYcfKHerT+kVKo2wd79S2LrviH7ZvmvKsUrn12FkJoqzMwDjPghf3VdFcgAAAABJRU5ErkJggg==); +} + +div.last { + width:16px; + height:16px; + display: inline-block; + background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAACXElEQVR4AWRTA4wcYRi9WI19se3buGEd1Yhr241qu7M427a5PNs2Z229zjtjkrf8nn6EANiBg0qESpBJOCbhArH+Wcb/ds9vJ+6TEPYq13I7RzctN3QMa7t7+8Wunj5R0zqgzagflz/PNN7mDGe3C2yQ96ep59509Y8OLqxYYHN64QsECVgdHswuGqFv7RmMrRh9w1lytguEkdwzMGZzuX3g4/T4YbR5IUqwu/0AIIl6oGnqtEWVDlEkDBRgL8am8wa5sHkJi2YPFkxrmDO5MWt0b4pU1OoHHyQvsE4oBWQ52ik5Y/NZtnhw+oMa+U2Lm+ST7+pxTdkLpuEzMD4PIdMgJ5cCx/TtQ1p2ZlSSTr1XS6hHrmERM6JLEqgDBVasXlgcPiyZnIhLL9KSS4ELnd29IheLAxSgI3H2swHy0kmcfFOHq4oeMN30igseXwDJqekiuSF86ejqoQA4wK4kn/mkA50VZVM48boGV+RrApPLLimFF3EJSZsCxxqa+7QmmxtWpw90IJmuTDS+6JQEqkEBLix3ZnjGBCEqdbOCLLVmRD45u4JgEHSQyDUgeXTBCYLkS0IPWM/rD6K2sQ9vVCWbixj6OG35do26aVC0uDZ3gsSReQeG5xwkbpJHJPe/EYmDl1XDa9u4cZCEvK43hZVq24LRDiYx2X1gmoklJ7ePZCm6EX+i0mwvhdKtg7T9KH9PbXqTmFk4aOgcweSCBVwTdu6bEFGm7sJ3IXbw+d/i7Ud572W6ET1x+11kmfynMkErKFTi779y8fOfKO3LPzny/0uYegMjM1GcnQFvhQh62IVc5gAAAABJRU5ErkJggg==); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-admin/src/main/resources/static/index.html ---------------------------------------------------------------------- diff --git a/gobblin-admin/src/main/resources/static/index.html b/gobblin-admin/src/main/resources/static/index.html index 98b10f9..3af5aaf 100644 --- a/gobblin-admin/src/main/resources/static/index.html +++ b/gobblin-admin/src/main/resources/static/index.html @@ -56,9 +56,12 @@
+
+ + - + + + + - - - + + + + + + @@ -177,6 +257,7 @@ + http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-admin/src/main/resources/static/js/collections/job-executions.js ---------------------------------------------------------------------- diff --git a/gobblin-admin/src/main/resources/static/js/collections/job-executions.js b/gobblin-admin/src/main/resources/static/js/collections/job-executions.js index cd46968..22b24f4 100644 --- a/gobblin-admin/src/main/resources/static/js/collections/job-executions.js +++ b/gobblin-admin/src/main/resources/static/js/collections/job-executions.js @@ -53,6 +53,12 @@ var app = app || {} paramString += '&' + key + '=' + params[key] } return paramString + }, + hasExecuted: function() { + var filtered = this.filter(function (e) { + return e.get("launchedTasks") > 0; + }); + return new JobExecutions(filtered) } }) http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-admin/src/main/resources/static/js/gobblin.js ---------------------------------------------------------------------- diff --git a/gobblin-admin/src/main/resources/static/js/gobblin.js b/gobblin-admin/src/main/resources/static/js/gobblin.js index 471fb84..05f43a9 100644 --- a/gobblin-admin/src/main/resources/static/js/gobblin.js +++ b/gobblin-admin/src/main/resources/static/js/gobblin.js @@ -18,7 +18,7 @@ var Gobblin = Gobblin || {} Gobblin.columnSchemas = { listJobs: [ - { name: 'Job Name', fn: 'getJobNameLink' }, + { name: 'Job Name', fn: 'getJobNameLink', sortInitialOrder: 'asc' }, { name: 'State', fn: 'getJobStateElem' }, { name: 'Schedule', fn: 'getSchedule' }, { name: 'Last Run Started', fn: 'getJobStartTime' }, @@ -26,7 +26,7 @@ Gobblin.columnSchemas = { { name: 'Extracted Records (most recent run)', fn: 'getRecordMetrics' } ], listByJobName: [ - { name: 'Job Id', fn: 'getJobIdLink' }, + { name: 'Job Id', fn: 'getJobIdLink', sortInitialOrder: 'desc' }, { name: 'State', fn: 'getJobStateElem' }, { name: 'Schedule', fn: 'getSchedule' }, { name: 'Completed/Launched Tasks', fn: 'getTaskRatio' }, @@ -36,7 +36,7 @@ Gobblin.columnSchemas = { { name: 'Extracted Records', fn: 'getRecordMetrics' } ], listTasksByJobId: [ - { name: 'Task Id', fn: 'getTaskId' }, + { name: 'Task Id', fn: 'getTaskId', sortInitialOrder: 'asc' }, { name: 'State', fn: 'getTaskStateElem' }, { name: 'Start Time', fn: 'getTaskStartTime' }, { name: 'End Time', fn: 'getTaskEndTime' }, @@ -65,5 +65,28 @@ Gobblin.stateMap = { 'FAILED': { color: Gobblin.colors.danger, class: 'danger' } } Gobblin.settings = { - restServerUrl: 'localhost:8080' + restServerUrl: 'localhost:8080', + hideJobsWithoutTasksByDefault: true, + refreshInterval: 30000 } +Gobblin.ViewManager = { + currentView : null, + showView : function(view) { + if (this.currentView !== null && this.currentView.cid != view.cid) { + if (this.currentView.onBeforeClose) { + this.currentView.onBeforeClose() + } + this.currentView.remove(); + } + this.currentView = view; + return view.render(); + } +} +Backbone.View.prototype._removeElement = function(){ + this.$el.empty().off(); +} +$(document).keyup(function(e) { + if (e.keyCode == 13) { + $(':focus').trigger('enter'); + } +}); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-admin/src/main/resources/static/js/router.js ---------------------------------------------------------------------- diff --git a/gobblin-admin/src/main/resources/static/js/router.js b/gobblin-admin/src/main/resources/static/js/router.js index 7992a18..6cbe513 100644 --- a/gobblin-admin/src/main/resources/static/js/router.js +++ b/gobblin-admin/src/main/resources/static/js/router.js @@ -28,13 +28,13 @@ var app = app || {} }, index: function () { - new app.OverView() + Gobblin.ViewManager.showView(new app.OverView()) }, job: function (name) { - new app.JobView(name) + Gobblin.ViewManager.showView(new app.JobView(name)) }, jobDetails: function (id) { - new app.JobExecutionView(id) + Gobblin.ViewManager.showView(new app.JobExecutionView(id)) } }) http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-admin/src/main/resources/static/js/views/job-execution-view.js ---------------------------------------------------------------------- diff --git a/gobblin-admin/src/main/resources/static/js/views/job-execution-view.js b/gobblin-admin/src/main/resources/static/js/views/job-execution-view.js index aa97039..adea8d0 100644 --- a/gobblin-admin/src/main/resources/static/js/views/job-execution-view.js +++ b/gobblin-admin/src/main/resources/static/js/views/job-execution-view.js @@ -20,32 +20,79 @@ var app = app || {} ;(function ($) { app.JobExecutionView = Backbone.View.extend({ - el: '#main-content', - + mainTemplate: _.template($('#main-template').html()), headerTemplate: _.template($('#header-template').html()), contentTemplate: _.template($('#job-execution-template').html()), - keyValueTemplate: _.template($('#key-value-template').html()), + summaryTemplate: _.template($('#summary-template').html()), events: { 'click #query-btn': '_fetchData' }, initialize: function (jobId) { - this.jobId = jobId - this.collection = app.jobExecutions - this.model = {} - - this.headerEl = this.$el.find('#header-container') - this.contentEl = this.$el.find('#content-container') + var self = this + self.setElement($('#main-content')) + self.jobId = jobId + self.collection = app.jobExecutions + if (Gobblin.settings.refreshInterval > 0) { + self.timer = setInterval(function() { + if (self.initialized) { + self._fetchData() + } + }, Gobblin.settings.refreshInterval) + } + self.listenTo(self.collection, 'reset', self.refreshData) + }, - this.render() + onBeforeClose: function() { + var self = this + if (self.timer) { + clearInterval(self.timer); + } + if (self.table) { + if (self.table.onBeforeClose) { + self.table.onBeforeClose() + } + self.table.remove() + } + if (self.propertiesTable) { + if (self.propertiesTable.onBeforeClose) { + self.propertiesTable.onBeforeClose() + } + self.propertiesTable.remove() + } + if (self.metricsTable) { + if (self.metricsTable.onBeforeClose) { + self.metricsTable.onBeforeClose() + } + self.metricsTable.remove() + } }, render: function () { - this.renderHeader() - this.contentEl.html(this.contentTemplate({})) + var self = this + self.$el.html(self.mainTemplate) + self.headerEl = self.$el.find('#header-container') + self.contentEl = self.$el.find('#content-container') + self.contentEl.html(self.contentTemplate({})) + self.renderHeader() + self.renderSummary() - this._fetchData() + self.table = new app.TableView({ + el: '#task-table-container', + collection: self.collection, + collectionResolver: function(c) { + if (c) { + return c.get(self.jobId).getTaskExecutions() + } + return {} + }, + columnSchema: 'listTasksByJobId' + }) + self.table.render() + self.initialized = true + + return self._fetchData() }, _fetchData: function () { @@ -56,65 +103,88 @@ var app = app || {} taskProperties: "", includeTaskMetrics: false } - self.collection.fetchCurrent('JOB_ID', self.jobId, opts).done(function () { + self.collection.fetchCurrent('JOB_ID', self.jobId, opts) + }, + + refreshData: function() { + var self = this + if (self.initialized) { self.model = self.collection.get(self.jobId) self.renderHeader(self.model.getJobStateMapped()) - self.renderSummary() - - self.table = new app.TableView({ - el: '#task-table-container', - collection: self.model.getTaskExecutions(), - columnSchema: 'listTasksByJobId', - includeJobToggle: false - }) - self.table.renderData() - }) + self.refreshSummary() + } }, renderHeader: function (status) { + var self = this var header = { title: 'Job Execution Details', - subtitle: this.jobId + subtitle: self.jobId } if (typeof status !== 'undefined') { header.highlightClass = status } - this.headerEl.html(this.headerTemplate({ header: header })) + self.headerEl.html(self.headerTemplate({ header: header })) }, renderSummary: function () { - this.generateKeyValue('About', this.getSummary(), '#important-key-value', false) - this.generateKeyValue('Job Properties', this.getProperties(), '#job-properties-key-value .well', true) - this.generateKeyValue('Metrics', this.getJobMetrics(), '#job-metrics-key-value .well', true) + var self = this + self.generateSummary('About', self.getSummary(), '#important-key-value', false) + self.propertiesTable = self.generateKeyValue('Job Properties', function(c) { return self.getProperties(c) }, '#job-properties-key-value', true) + self.metricsTable = self.generateKeyValue('Metrics', function(c) { return self.getJobMetrics(c) }, '#job-metrics-key-value', true) + }, + refreshSummary: function () { + var self = this + self.generateSummary('About', self.getSummary(), '#important-key-value', false) }, - generateKeyValue: function (title, keyValuePairs, elemId, center) { - this.$el.find(elemId).html(this.keyValueTemplate({ + generateSummary: function (title, keyValuePairs, elemId, center) { + var self = this + self.$el.find(elemId).html(self.summaryTemplate({ title: title, pairs: keyValuePairs, center: center })) }, + generateKeyValue: function (title, keyValuePairResolver, elemId, center) { + var self = this + var propertiesTable = new app.KeyValueTableView({ + el: elemId, + title: title, + center: center, + collection: self.collection, + collectionResolver: keyValuePairResolver + }) + propertiesTable.render() + return propertiesTable + }, getSummary: function () { + var self = this return { - 'Job Name': this.model.getJobNameLink(), - 'Job Id': this.model.getJobIdLink(), - 'State': this.model.getJobStateElem(), - 'Completed/Launched Tasks': this.model.getTaskRatio(), - 'Start Time': this.model.getJobStartTime(), - 'End Time': this.model.getJobEndTime(), - 'Duration (seconds)': this.model.getDurationInSeconds(), - 'Launcher Type': this.model.getLauncherType() + 'Job Name': self.model ? self.model.getJobNameLink() : '', + 'Job Id': self.model ? self.model.getJobIdLink() : '', + 'State': self.model ? self.model.getJobStateElem() : '', + 'Completed/Launched Tasks': self.model ? self.model.getTaskRatio() : '', + 'Start Time': self.model ? self.model.getJobStartTime() : '', + 'End Time': self.model ? self.model.getJobEndTime() : '', + 'Duration (seconds)': self.model ? self.model.getDurationInSeconds() : '', + 'Launcher Type': self.model ? self.model.getLauncherType() : '' } }, - getProperties: function () { - if (this.model.hasProperties()) { - return this.model.attributes.jobProperties + getProperties: function (collection) { + var self = this + var model = collection.get(self.jobId) + if (model && model.hasProperties()) { + return _.object(_.map(_.sortBy(_.keys(model.attributes.jobProperties)), function(key) { + return [key, model.attributes.jobProperties[key]] + })) } return {} }, - getJobMetrics: function () { - if (this.model.attributes.metrics) { - var jobMetrics = this.model.attributes.metrics.filter(function (metric) { + getJobMetrics: function (collection) { + var self = this + var model = collection.get(self.jobId) + if (model && model.attributes.metrics) { + var jobMetrics = model.attributes.metrics.filter(function (metric) { return metric.group === 'JOB' }) http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-admin/src/main/resources/static/js/views/job-view.js ---------------------------------------------------------------------- diff --git a/gobblin-admin/src/main/resources/static/js/views/job-view.js b/gobblin-admin/src/main/resources/static/js/views/job-view.js index eb74af5..8b1cdc9 100644 --- a/gobblin-admin/src/main/resources/static/js/views/job-view.js +++ b/gobblin-admin/src/main/resources/static/js/views/job-view.js @@ -20,39 +20,64 @@ var app = app || {} ;(function ($) { app.JobView = Backbone.View.extend({ - el: '#main-content', - + mainTemplate: _.template($('#main-template').html()), headerTemplate: _.template($('#header-template').html()), contentTemplate: _.template($('#job-template').html()), chartTemplate: _.template($('#chart-template').html()), - keyValueTemplate: _.template($('#key-value-template').html()), + summaryTemplate: _.template($('#summary-template').html()), events: { - 'click #query-btn': '_fetchData' + 'click #query-btn': '_fetchData', + 'enter #results-limit': '_fetchData' }, initialize: function (jobName) { - this.jobName = jobName - this.collection = app.jobExecutions - - this.headerEl = this.$el.find('#header-container') - this.contentEl = this.$el.find('#content-container') + var self = this + self.setElement($('#main-content')) + self.jobName = jobName + self.collection = app.jobExecutions + if (Gobblin.settings.refreshInterval > 0) { + self.timer = setInterval(function() { + if (self.initialized) { + self._fetchData() + } + }, Gobblin.settings.refreshInterval) + } + self.listenTo(self.collection, 'reset', self.refreshData) + }, - this.render() + onBeforeClose: function() { + var self = this + if (self.timer) { + clearInterval(self.timer); + } + if (self.table) { + if (self.table.onBeforeClose) { + self.table.onBeforeClose() + } + self.table.remove() + } }, render: function () { var self = this - self.renderHeader() + self.$el.html(self.mainTemplate) + self.headerEl = self.$el.find('#header-container') + self.contentEl = self.$el.find('#content-container') self.contentEl.html(self.contentTemplate({})) + self.renderHeader() + self.renderSummary() self.table = new app.TableView({ el: '#job-table-container', collection: self.collection, columnSchema: 'listByJobName', - includeJobToggle: false + includeJobToggle: false, + includeJobsWithTasksToggle: true, }) + self.table.render() + self.initialized = true self._fetchData() }, @@ -60,38 +85,59 @@ var app = app || {} _fetchData: function () { var self = this + var includeJobsWithoutTasks = $('#list-jobs-with-tasks-toggle .active input').val() === "ALL" + var opts = { limit: self.table.getLimit(), includeTaskExecutions: false, includeTaskMetrics: false, + includeJobsWithoutTasks: includeJobsWithoutTasks, jobProperties: 'job.description,job.runonce,job.schedule', taskProperties: '' } - self.collection.fetchCurrent('JOB_NAME', self.jobName, opts).done(function () { - self.renderHeader(self.collection.first().getJobStateMapped()) - self.renderSummary() - self.table.renderData() - }) + self.collection.fetchCurrent('JOB_NAME', self.jobName, opts) }, renderHeader: function (status) { + var self = this var header = { title: 'Job Information', - subtitle: this.jobName + subtitle: self.jobName } if (typeof status !== 'undefined') { header.highlightClass = status } - this.headerEl.html(this.headerTemplate({ header: header })) + self.headerEl.html(self.headerTemplate({ header: header })) + }, + refreshData: function() { + var self = this + if (self.initialized) { + self.renderHeader(self.collection.last().getJobStateMapped()) + if (self.durationChart !== undefined || self.recordsChart !== undefined) { + var jobData = self.getDurationAndRecordsRead() + if (self.durationChart !== undefined) { + self.durationChart.data.labels = jobData.labels + self.durationChart.data.datasets[0].data = jobData.durations + self.durationChart.update(); + } + if (self.recordsChart !== undefined) { + self.recordsChart.data.labels = jobData.labels + self.recordsChart.data.datasets[0].data = jobData.recordsRead + self.recordsChart.update(); + } + } + self.generateSummary('Status', self.getStatusReport(), '#status-key-value') + } }, - renderSummary: function () { - var jobData = this.getDurationAndRecordsRead() - this.generateNewLineChart('Job Duration', jobData.labels, jobData.durations, '#duration-chart') - this.generateNewLineChart('Records Read', jobData.labels, jobData.recordsRead, '#records-chart') - this.generateKeyValue('Status', this.getStatusReport(), '#status-key-value') + var self = this + var jobData = self.getDurationAndRecordsRead() + self.durationChart = self.generateNewLineChart('Job Duration', jobData.labels, jobData.durations, '#duration-chart') + self.recordsChart = self.generateNewLineChart('Records Read', jobData.labels, jobData.recordsRead, '#records-chart') + self.generateSummary('Status', self.getStatusReport(), '#status-key-value') }, getDurationAndRecordsRead: function (maxExecutions) { + var self = this maxExecutions = maxExecutions || 10 var values = { @@ -99,9 +145,10 @@ var app = app || {} durations: [], recordsRead: [] } - var max = this.collection.size() < maxExecutions ? this.collection.size() : maxExecutions - for (var i = max - 1; i >= 0; i--) { - var execution = this.collection.at(i) + var executedCollection = self.collection.hasExecuted(); + var min = executedCollection.size() < maxExecutions ? 0 : executedCollection.size() - maxExecutions; + for (var i = min; i < executedCollection.size(); i++) { + var execution = executedCollection.at(i) values.labels.push(execution.getJobStartTime()) var time = execution.getDurationInSeconds() === '-' ? 0 : execution.getDurationInSeconds() values.durations.push(time) @@ -110,9 +157,10 @@ var app = app || {} return values }, getStatusReport: function () { + var self = this var statuses = {} - for (var i = 0; i < this.collection.size(); i++) { - var execution = this.collection.at(i) + for (var i = 0; i < self.collection.size(); i++) { + var execution = self.collection.at(i) statuses[execution.getJobState()] = (statuses[execution.getJobState()] || 0) + 1 } return statuses @@ -130,28 +178,48 @@ var app = app || {} } var chartData = { - labels: labels, - datasets: [ - $.extend({ - label: title, - data: data - }, lineFormat) - ] + type: 'line', + data: { + labels: labels, + datasets: [ + $.extend({ + label: title, + data: data + }, lineFormat) + ] + }, + options: { + legend: { + display: false + }, + scales: { + yAxes: [{ + ticks: { + beginAtZero: true, + userCallback: function(label, index, labels) { + if (Math.floor(label) === label) { + return label; + } + } + } + }] + } + } } var chartElem = self.$el.find(elemId) - chartElem.html(this.chartTemplate({ + chartElem.html(self.chartTemplate({ title: title, height: 450, width: 600 })) var ctx = chartElem.find('.chart-canvas')[0].getContext('2d') - return new Chart(ctx).Line(chartData, { responsive: true }) + return new Chart(ctx, chartData) }, - generateKeyValue: function (title, keyValuePairs, elemId) { + generateSummary: function (title, keyValuePairs, elemId) { var self = this keyValuePairs = self.pseudoSortStates(keyValuePairs) - self.$el.find(elemId).html(self.keyValueTemplate({ title: title, pairs: keyValuePairs, center: true })) + self.$el.find(elemId).html(self.summaryTemplate({ title: title, pairs: keyValuePairs, center: true })) }, pseudoSortStates: function (data) { var newData = {} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-admin/src/main/resources/static/js/views/key-value-table-view.js ---------------------------------------------------------------------- diff --git a/gobblin-admin/src/main/resources/static/js/views/key-value-table-view.js b/gobblin-admin/src/main/resources/static/js/views/key-value-table-view.js new file mode 100644 index 0000000..6dded94 --- /dev/null +++ b/gobblin-admin/src/main/resources/static/js/views/key-value-table-view.js @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* global Backbone, _, jQuery, Gobblin */ +var app = app || {} + +;(function ($) { + app.KeyValueTableView = Backbone.View.extend({ + tableTemplate: _.template($('#key-value-table-template').html()), + tableBodyTemplate: _.template($('#key-value-table-body-template').html()), + + initialize: function (options) { + var self = this + self.setElement(options.el) + self.title = options.title + self.center = options.center + self.collection = options.collection + self.collectionResolver = options.collectionResolver || function(c) { return c } + + self.listenTo(self.collection, 'reset', self.refreshData); + }, + + render: function () { + // Data should be fetched by parent view before calling this function + var self = this + self.$el.find('#key-value-table-container').html(self.tableTemplate({ + title: self.title, + center: self.center, + data: [] + })) + + // TODO attach elsewhere? + self.$el.find('#key-value-table').tablesorter({ + theme: 'bootstrap', + headerTemplate: '{content} {icon}', + widthFixed: true, + widgets: [ 'uitheme', 'filter' ] + }) + .tablesorterPager({ + container: self.$el.find('#key-value-table-pager'), + output: '{startRow} - {endRow} / {filteredRows} ({totalRows})', + fixedHeight: false, + removeRows: true + }); + self.initialized = true + }, + + refreshData: function() { + var self = this + if (self.initialized) { + var table = self.$el.find('#key-value-table-container table') + var page = table[0].config.pager.page + 1 + var data = self.collectionResolver(self.collection) + var tableBody = table.find('tbody') + tableBody.empty() + tableBody.html(self.tableBodyTemplate({ data: data })) + table.trigger('update', [true]) + table.trigger('pagerUpdate', page) + } + } + }) +})(jQuery) http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-admin/src/main/resources/static/js/views/over-view.js ---------------------------------------------------------------------- diff --git a/gobblin-admin/src/main/resources/static/js/views/over-view.js b/gobblin-admin/src/main/resources/static/js/views/over-view.js index 8ba4ed9..0e9ecf4 100644 --- a/gobblin-admin/src/main/resources/static/js/views/over-view.js +++ b/gobblin-admin/src/main/resources/static/js/views/over-view.js @@ -20,59 +20,83 @@ var app = app || {} ;(function ($) { app.OverView = Backbone.View.extend({ - el: '#main-content', - + mainTemplate: _.template($('#main-template').html()), headerTemplate: _.template($('#header-template').html()), contentTemplate: _.template($('#list-all-template').html()), events: { - 'click #query-btn': '_fetchData' + 'click #query-btn': '_fetchData', + 'enter #results-limit': '_fetchData' }, initialize: function () { - this.collection = app.jobExecutions - - this.headerEl = this.$el.find('#header-container') - this.contentEl = this.$el.find('#content-container') + var self = this + self.setElement($('#main-content')) + self.collection = app.jobExecutions + if (Gobblin.settings.refreshInterval > 0) { + self.timer = setInterval(function () { + if (self.initialized) { + self._fetchData() + } + }, Gobblin.settings.refreshInterval); + } + }, - this.render() + onBeforeClose: function() { + var self = this + if (self.timer) { + clearInterval(self.timer); + } + if (self.table) { + if (self.table.onBeforeClose) { + self.table.onBeforeClose() + } + self.table.remove() + } }, render: function () { var self = this + self.$el.html(self.mainTemplate) + self.headerEl = self.$el.find('#header-container') self.headerEl.html(self.headerTemplate({ header: { title: 'Gobblin Jobs' } })) + self.contentEl = self.$el.find('#content-container') self.contentEl.html(self.contentTemplate({})) self.table = new app.TableView({ el: '#list-all-table-container', collection: self.collection, columnSchema: 'listJobs', - includeJobToggle: true + includeJobToggle: true, + includeJobsWithTasksToggle: true }) + self.table.render() + self.initialized = true - self._fetchData() + return self._fetchData() }, _fetchData: function () { var self = this + var includeJobsWithoutTasks = $('#list-jobs-with-tasks-toggle .active input').val() === "ALL" + var opts = { limit: self.table.getLimit(), includeTaskExecutions: false, includeJobMetrics: false, includeTaskMetrics: false, + includeJobsWithoutTasks: includeJobsWithoutTasks, jobProperties: 'job.description,job.runonce,job.schedule', taskProperties: '' } var id = $('#list-jobs-toggle .active input').val() - self.collection.fetchCurrent('LIST_TYPE', id, opts).done(function () { - self.table.renderData() - }) + self.collection.fetchCurrent('LIST_TYPE', id, opts) } }) })(jQuery) http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-admin/src/main/resources/static/js/views/table-view.js ---------------------------------------------------------------------- diff --git a/gobblin-admin/src/main/resources/static/js/views/table-view.js b/gobblin-admin/src/main/resources/static/js/views/table-view.js index a8e4b04..30f2ca3 100644 --- a/gobblin-admin/src/main/resources/static/js/views/table-view.js +++ b/gobblin-admin/src/main/resources/static/js/views/table-view.js @@ -22,15 +22,19 @@ var app = app || {} app.TableView = Backbone.View.extend({ tableControlTemplate: _.template($('#table-control-template').html()), tableTemplate: _.template($('#table-template').html()), + tableBodyTemplate: _.template($('#table-body-template').html()), initialize: function (options) { - this.setElement(options.el) - this.collection = options.collection - this.columnSchema = options.columnSchema - this.includeJobToggle = options.includeJobToggle || false - this.resultsLimit = options.resultsLimit || 100 - - this.render() + var self = this + self.setElement(options.el) + self.collectionResolver = options.collectionResolver || function(c) { return c } + self.collection = options.collection + self.columnSchema = options.columnSchema + self.includeJobToggle = options.includeJobToggle || false + self.includeJobsWithTasksToggle = options.includeJobsWithTasksToggle || false + self.hideJobsWithoutTasksByDefault = options.hideJobsWithoutTasksByDefault || Gobblin.settings.hideJobsWithoutTasksByDefault || false + self.resultsLimit = options.resultsLimit || 100 + self.listenTo(self.collection, 'reset', self.refreshData); }, render: function () { @@ -38,40 +42,82 @@ var app = app || {} self.$el.find('#table-control-container').html(self.tableControlTemplate({ includeJobToggle: self.includeJobToggle, + includeJobsWithTasksToggle: self.includeJobsWithTasksToggle, + hideJobsWithoutTasksByDefault: self.hideJobsWithoutTasksByDefault, resultsLimit: self.resultsLimit })) - }, - renderData: function () { - // Data should be fetched by parent view before calling this function - var self = this var columnHeaders = Gobblin.columnSchemas[self.columnSchema] self.$el.find('#table-container').html(self.tableTemplate({ includeJobToggle: self.includeJobToggle, - columnHeaders: columnHeaders, - data: self.collection.map(function (execution) { - var row = [] + includeJobsWithTasksToggle: self.includeJobsWithTasksToggle, + hideJobsWithoutTasksByDefault: self.hideJobsWithoutTasksByDefault, + columnHeaders: columnHeaders + })) - for (var i in columnHeaders) { - row.push(execution[columnHeaders[i].fn]()) - } - return row - }) + self.$el.find('#table-container table tbody').html(self.tableBodyTemplate({ + data: [] })) + var sortList = [] + for (var i in columnHeaders) { + if ('sortInitialOrder' in columnHeaders[i]) { + if (columnHeaders[i].sortInitialOrder === 'asc') { + sortList.push([parseInt(i), 0]) + } else if (columnHeaders[i].sortInitialOrder === 'desc') { + sortList.push([parseInt(i), 1]) + } + } + } + if (sortList.length == 0) { + sortList.push([0,0]) + } + // TODO attach elsewhere? - $('table').tablesorter({ + self.$el.find('#jobs-table').tablesorter({ theme: 'bootstrap', headerTemplate: '{content} {icon}', - widgets: [ 'uitheme' ] + widthFixed: true, + widgets: [ 'uitheme', 'filter' ], + sortList: sortList }) + .tablesorterPager({ + container: self.$el.find('#jobs-table-pager'), + output: '{startRow} - {endRow} / {filteredRows} ({totalRows})', + fixedHeight: false, + removeRows: true + }); + self.initialized = true + }, + + refreshData: function() { + var self = this + if (self.initialized) { + self.tableCollection = self.collectionResolver(self.collection) + var columnHeaders = Gobblin.columnSchemas[self.columnSchema] + var table = self.$el.find('#table-container table') + var page = table[0].config.pager.page + 1 + var data = self.tableCollection.map(function (execution) { + var row = [] + for (var i in columnHeaders) { + row.push(execution[columnHeaders[i].fn]()) + } + return row + }) + var tableBody = table.find('tbody') + tableBody.empty() + tableBody.html(self.tableBodyTemplate({ data: data })) + table.trigger('update', [true]) + table.trigger('pagerUpdate', page) + } }, getLimit: function () { - var limitElem = this.$el.find('#results-limit') - if (limitElem.val().length === 0) { - return limitElem.attr('placeholder') + var self = this + var limitElem = self.$el.find('#results-limit') + if (limitElem.val() === undefined || limitElem.val().length === 0) { + return self.resultsLimit } return limitElem.val() } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java b/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java index 1e6e1bb..c20a7d3 100644 --- a/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java +++ b/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java @@ -684,6 +684,10 @@ public class ConfigurationKeys { public static final String DEFAULT_ADMIN_SERVER_HOST = "localhost"; public static final String ADMIN_SERVER_PORT_KEY = "admin.server.port"; public static final String DEFAULT_ADMIN_SERVER_PORT = "8000"; + public static final String ADMIN_SERVER_HIDE_JOBS_WITHOUT_TASKS_BY_DEFAULT_KEY = "admin.server.hide_jobs_without_tasks_by_default.enabled"; + public static final String DEFAULT_ADMIN_SERVER_HIDE_JOBS_WITHOUT_TASKS_BY_DEFAULT = "false"; + public static final String ADMIN_SERVER_REFRESH_INTERVAL_KEY = "admin.server.refresh_interval"; + public static final long DEFAULT_ADMIN_SERVER_REFRESH_INTERVAL = 30000; /** * Kafka job configurations. http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java ---------------------------------------------------------------------- diff --git a/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java b/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java index 6909afc..7ecb4a3 100644 --- a/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java +++ b/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java @@ -120,14 +120,14 @@ public class DatabaseJobHistoryStoreV101 implements VersionedDatabaseJobHistoryS private static final String LIST_DISTINCT_JOB_EXECUTION_QUERY_TEMPLATE = "SELECT j.job_id FROM gobblin_job_executions j, " + "(SELECT MAX(last_modified_ts) AS most_recent_ts, job_name " - + "FROM gobblin_job_executions GROUP BY job_name) max_results " + + "FROM gobblin_job_executions%s GROUP BY job_name) max_results " + "WHERE j.job_name = max_results.job_name AND j.last_modified_ts = max_results.most_recent_ts"; private static final String LIST_RECENT_JOB_EXECUTION_QUERY_TEMPLATE = - "SELECT job_id FROM gobblin_job_executions"; + "SELECT job_id FROM gobblin_job_executions%s"; private static final String JOB_NAME_QUERY_BY_TABLE_STATEMENT_TEMPLATE = "SELECT j.job_name FROM gobblin_job_executions j, gobblin_task_executions t " - + "WHERE j.job_id=t.job_id AND %s GROUP BY j.job_name"; + + "WHERE j.job_id=t.job_id AND %s%s GROUP BY j.job_name"; private static final String JOB_ID_QUERY_BY_JOB_NAME_STATEMENT_TEMPLATE = "SELECT job_id FROM gobblin_job_executions WHERE job_name=?"; @@ -150,6 +150,8 @@ public class DatabaseJobHistoryStoreV101 implements VersionedDatabaseJobHistoryS private static final String TASK_PROPERTY_QUERY_STATEMENT_TEMPLATE = "SELECT job_id,p.task_id,property_key,property_value FROM gobblin_task_properties p JOIN gobblin_task_executions t ON t.task_id = p.task_id WHERE job_id IN (%s)"; + private static final String FILTER_JOBS_WITH_TASKS = "(`state` != 'COMMITTED' OR launched_tasks > 0)"; + private DataSource dataSource; @Override @@ -706,6 +708,10 @@ public class DatabaseJobHistoryStoreV101 implements VersionedDatabaseJobHistoryS } } + if (!query.isIncludeJobsWithoutTasks()) { + jobIdByNameQuery += " AND " + FILTER_JOBS_WITH_TASKS; + } + // Add ORDER BY jobIdByNameQuery += " ORDER BY created_ts DESC"; @@ -735,8 +741,14 @@ public class DatabaseJobHistoryStoreV101 implements VersionedDatabaseJobHistoryS Filter tableFilter = constructTableFilter(query.getId().getTable()); + String jobsWithoutTaskFilter = ""; + if (!query.isIncludeJobsWithoutTasks()) { + jobsWithoutTaskFilter = " AND " + FILTER_JOBS_WITH_TASKS; + } + // Construct the query for job names by table definition - String jobNameByTableQuery = String.format(JOB_NAME_QUERY_BY_TABLE_STATEMENT_TEMPLATE, tableFilter.getFilter()); + String jobNameByTableQuery = String.format(JOB_NAME_QUERY_BY_TABLE_STATEMENT_TEMPLATE, tableFilter.getFilter(), + jobsWithoutTaskFilter); List jobExecutionInfos = Lists.newArrayList(); // Query job names by table definition @@ -777,6 +789,12 @@ public class DatabaseJobHistoryStoreV101 implements VersionedDatabaseJobHistoryS } else { listJobExecutionsQuery = LIST_RECENT_JOB_EXECUTION_QUERY_TEMPLATE; } + + String jobsWithoutTaskFilter = ""; + if (!query.isIncludeJobsWithoutTasks()) { + jobsWithoutTaskFilter = " WHERE " + FILTER_JOBS_WITH_TASKS; + } + listJobExecutionsQuery = String.format(listJobExecutionsQuery, jobsWithoutTaskFilter); listJobExecutionsQuery += " ORDER BY last_modified_ts DESC"; try (PreparedStatement queryStatement = connection.prepareStatement(listJobExecutionsQuery)) { http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/gobblin/rest/JobExecutionQuery.pdsc ---------------------------------------------------------------------- diff --git a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/gobblin/rest/JobExecutionQuery.pdsc b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/gobblin/rest/JobExecutionQuery.pdsc index 7fe55d7..b14d779 100644 --- a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/gobblin/rest/JobExecutionQuery.pdsc +++ b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/gobblin/rest/JobExecutionQuery.pdsc @@ -72,6 +72,13 @@ "optional": true, "default": true, "doc": "true/false if the response should include task executions (default: true)" + }, + { + "name": "includeJobsWithoutTasks", + "type": "boolean", + "optional": true, + "default": true, + "doc": "true/false if the response should include jobs that did not launch tasks (default: true)" } ] } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b12c3538/gobblin-rest-service/gobblin-rest-api/src/main/snapshot/gobblin.rest.jobExecutions.snapshot.json ---------------------------------------------------------------------- diff --git a/gobblin-rest-service/gobblin-rest-api/src/main/snapshot/gobblin.rest.jobExecutions.snapshot.json b/gobblin-rest-service/gobblin-rest-api/src/main/snapshot/gobblin.rest.jobExecutions.snapshot.json index 7a608e1..594128c 100644 --- a/gobblin-rest-service/gobblin-rest-api/src/main/snapshot/gobblin.rest.jobExecutions.snapshot.json +++ b/gobblin-rest-service/gobblin-rest-api/src/main/snapshot/gobblin.rest.jobExecutions.snapshot.json @@ -307,6 +307,12 @@ "doc" : "true/false if the response should include task executions (default: true)", "default" : true, "optional" : true + }, { + "name" : "includeJobsWithoutTasks", + "type" : "boolean", + "doc" : "true/false if the response should include jobs that did not launch tasks (default: true)", + "default" : true, + "optional" : true } ] }, { "type" : "record", @@ -319,10 +325,10 @@ } } ], "schema" : { - "schema" : "gobblin.rest.JobExecutionQueryResult", - "path" : "/jobExecutions", "name" : "jobExecutions", "namespace" : "gobblin.rest", + "path" : "/jobExecutions", + "schema" : "gobblin.rest.JobExecutionQueryResult", "doc" : "A Rest.li resource for serving queries of Gobblin job executions.\n\ngenerated from: gobblin.rest.JobExecutionInfoResource", "collection" : { "identifier" : { @@ -330,12 +336,12 @@ "type" : "gobblin.rest.JobExecutionQuery", "params" : "com.linkedin.restli.common.EmptyRecord" }, + "supports" : [ "batch_get", "get" ], "methods" : [ { "method" : "get" }, { "method" : "batch_get" } ], - "supports" : [ "batch_get", "get" ], "entity" : { "path" : "/jobExecutions/{jobExecutionsId}" }