spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From abellina <...@git.apache.org>
Subject [GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Date Tue, 11 Sep 2018 22:19:13 GMT
Github user abellina commented on a diff in the pull request:

    https://github.com/apache/spark/pull/21688#discussion_r216732491
  
    --- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js ---
    @@ -0,0 +1,807 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *    http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +$(document).ajaxStop($.unblockUI);
    +$(document).ajaxStart(function () {
    +    $.blockUI({message: '<h3>Loading Stage Page...</h3>'});
    +});
    +
    +$.extend( $.fn.dataTable.ext.type.order, {
    +    "file-size-pre": ConvertDurationString,
    +
    +    "file-size-asc": function ( a, b ) {
    +        a = ConvertDurationString( a );
    +        b = ConvertDurationString( b );
    +        return ((a < b) ? -1 : ((a > b) ? 1 : 0));
    +    },
    +
    +    "file-size-desc": function ( a, b ) {
    +        a = ConvertDurationString( a );
    +        b = ConvertDurationString( b );
    +        return ((a < b) ? 1 : ((a > b) ? -1 : 0));
    +    }
    +} );
    +
    +// This function will only parse the URL under certain format
    +// e.g. https://domain:50509/history/application_1502220952225_59143/stages/stage/?id=0&attempt=0
    +function stageEndPoint(appId) {
    +    var words = document.baseURI.split('/');
    +    var words2 = document.baseURI.split('?');
    +    var ind = words.indexOf("proxy");
    +    if (ind > 0) {
    +        var appId = words[ind + 1];
    +        var stageIdLen = words2[1].indexOf('&');
    +        var stageId = words2[1].substr(3, stageIdLen - 3);
    +        var newBaseURI = words.slice(0, ind + 2).join('/');
    +        return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
    +    }
    +    ind = words.indexOf("history");
    +    if (ind > 0) {
    +        var appId = words[ind + 1];
    +        var attemptId = words[ind + 2];
    +        var stageIdLen = words2[1].indexOf('&');
    +        var stageId = words2[1].substr(3, stageIdLen - 3);
    +        var newBaseURI = words.slice(0, ind).join('/');
    +        if (isNaN(attemptId) || attemptId == "0") {
    +            return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
    +        } else {
    +            return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/stages/"
+ stageId;
    +        }
    +    }
    +    var stageIdLen = words2[1].indexOf('&');
    +    var stageId = words2[1].substr(3, stageIdLen - 3);
    +    return location.origin + "/api/v1/applications/" + appId + "/stages/" + stageId;
    +}
    +
    +function getColumnNameForTaskMetricSummary(columnKey) {
    +    if(columnKey == "executorRunTime") {
    +        return "Duration";
    +    }
    +    else if(columnKey == "jvmGcTime") {
    +        return "GC Time";
    +    }
    +    else if(columnKey == "gettingResultTime") {
    +        return "Getting Result Time";
    +    }
    +    else if(columnKey == "inputMetrics") {
    +        return "Input Size / Records";
    +    }
    +    else if(columnKey == "outputMetrics") {
    +        return "Output Size / Records";
    +    }
    +    else if(columnKey == "peakExecutionMemory") {
    +        return "Peak Execution Memory";
    +    }
    +    else if(columnKey == "resultSerializationTime") {
    +        return "Result Serialization Time";
    +    }
    +    else if(columnKey == "schedulerDelay") {
    +        return "Scheduler Delay";
    +    }
    +    else if(columnKey == "diskBytesSpilled") {
    +        return "Shuffle spill (disk)";
    +    }
    +    else if(columnKey == "memoryBytesSpilled") {
    +        return "Shuffle spill (memory)";
    +    }
    +    else if(columnKey == "shuffleReadMetrics") {
    +        return "Shuffle Read Size / Records";
    +    }
    +    else if(columnKey == "shuffleWriteMetrics") {
    +        return "Shuffle Write Size / Records";
    +    }
    +    else if(columnKey == "executorDeserializeTime") {
    +        return "Task Deserialization Time";
    +    }
    +    return "NA";
    +}
    +
    +$(document).ready(function () {
    +    setDataTableDefaults();
    +
    +    $("#showAdditionalMetrics").append(
    +        "<div><a id='taskMetric'>" +
    +        "<span class='expand-input-rate-arrow arrow-closed' id='arrowtoggle1'></span>"
+
    +        " Show Additional Metrics" +
    +        "</a></div>" +
    +        "<div class='container-fluid' id='toggle-metrics' hidden>" +
    +        "<div><input type='checkbox' class='toggle-vis' id='box-0' data-column='0'>
Select All</div>" +
    +        "<div><input type='checkbox' class='toggle-vis' id='box-11' data-column='11'>
Scheduler Delay</div>" +
    +        "<div><input type='checkbox' class='toggle-vis' id='box-12' data-column='12'>
Task Deserialization Time</div>" +
    +        "<div><input type='checkbox' class='toggle-vis' id='box-13' data-column='13'>
Shuffle Read Blocked Time</div>" +
    +        "<div><input type='checkbox' class='toggle-vis' id='box-14' data-column='14'>
Shuffle Remote Reads</div>" +
    +        "<div><input type='checkbox' class='toggle-vis' id='box-15' data-column='15'>
Result Serialization Time</div>" +
    +        "<div><input type='checkbox' class='toggle-vis' id='box-16' data-column='16'>
Getting Result Time</div>" +
    +        "<div><input type='checkbox' class='toggle-vis' id='box-17' data-column='17'>
Peak Execution Memory</div>" +
    +        "</div>");
    +
    +    tasksSummary = $("#active-tasks");
    +    getStandAloneAppId(function (appId) {
    +
    +        var endPoint = stageEndPoint(appId);
    +        $.getJSON(endPoint, function(response, status, jqXHR) {
    +
    +            // prepare data for task aggregated metrics table
    +            indices = Object.keys(response[0].executorSummary);
    +            var task_summary_table = [];
    +            indices.forEach(function (ix) {
    +               response[0].executorSummary[ix].id = ix;
    +               task_summary_table.push(response[0].executorSummary[ix]);
    +            });
    +
    +            // prepare data for accumulatorUpdates
    +            var indices = Object.keys(response[0].accumulatorUpdates);
    +            var accumulator_table_all = [];
    +            var accumulator_table = [];
    +            indices.forEach(function (ix) {
    +               accumulator_table_all.push(response[0].accumulatorUpdates[ix]);
    +            });
    +
    +            accumulator_table_all.forEach(function (x){
    +                var name = (x.name).toString();
    +                if(name.includes("internal.") == false){
    +                    accumulator_table.push(x);
    +                }
    +            });
    +
    +            // rendering the UI page
    +            var data = {"executors": response};
    +            $.get(createTemplateURI(appId, "stagespage"), function(template) {
    +                tasksSummary.append(Mustache.render($(template).filter("#stages-summary-template").html(),
data));
    +
    +                $("#taskMetric").click(function(){
    +                    $("#arrowtoggle1").toggleClass("arrow-open arrow-closed");
    +                    $("#toggle-metrics").toggle();
    +                });
    +
    +                $("#aggregatedMetrics").click(function(){
    +                    $("#arrowtoggle2").toggleClass("arrow-open arrow-closed");
    +                    $("#toggle-aggregatedMetrics").toggle();
    +                });
    +
    +                var task_metrics_table = [];
    +                var stageAttemptId = getStageAttemptId();
    +                $.getJSON(stageEndPoint(appId) + "/"+stageAttemptId+"/taskSummary?quantiles=0,0.25,0.5,0.75,1.0",
function(response1, status, jqXHR) {
    +                    var taskMetricIndices = Object.keys(response1);
    +                    taskMetricIndices.forEach(function (ix) {
    +                        var columnName = getColumnNameForTaskMetricSummary(ix);
    +                        if (columnName == "Shuffle Read Size / Records") {
    +                            var row1 = {
    +                                "metric": columnName,
    +                                "data": response1[ix]
    +                            };
    +                            var row2 = {
    +                                "metric": "Shuffle Read Blocked Time",
    +                                "data": response1[ix]
    +                            };
    +                            var row3 = {
    +                                "metric": "Shuffle Remote Reads",
    +                                "data": response1[ix]
    +                            };
    +                            task_metrics_table.push(row1);
    +                            task_metrics_table.push(row2);
    +                            task_metrics_table.push(row3);
    +                        }
    +                        else if (columnName != "NA") {
    +                            var row = {
    +                                "metric": columnName,
    +                                "data": response1[ix]
    +                            };
    +                            task_metrics_table.push(row);
    +                        }
    +                    });
    +
    +                    var taskMetricsTable = "#summary-metrics-table";
    +                    var task_conf = {
    +                        "data": task_metrics_table,
    +                        "columns": [
    +                            {data : 'metric'},
    +                            {
    +                                data: function (row, type) {
    +                                    if (row.metric == 'Input Size / Records') {
    --- End diff --
    
    for this chained if, and the ones below, change to a switch?


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org


Mime
View raw message