Mailing-List: contact issues-help@drill.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@drill.apache.org
Date: Thu, 23 Feb 2017 20:16:45 +0000 (UTC)
From: "ASF GitHub Bot (JIRA)" <jira@apache.org>
To: issues@drill.apache.org
Message-ID: <JIRA.13034855.1484350749000.47216.1487881005514@Atlassian.JIRA>
In-Reply-To: <JIRA.13034855.1484350749000@Atlassian.JIRA>
References: <JIRA.13034855.1484350749000@Atlassian.JIRA> <JIRA.13034855.1484350749380@jira-lw-us.apache.org>
Subject: [jira] [Commented] (DRILL-5195) Publish Operator and MajorFragment
 Stats in Profile page
MIME-Version: 1.0
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: 7bit
archived-at: Thu, 23 Feb 2017 20:16:53 -0000


    [ https://issues.apache.org/jira/browse/DRILL-5195?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15881155#comment-15881155 ] 

ASF GitHub Bot commented on DRILL-5195:
---------------------------------------

Github user paul-rogers commented on a diff in the pull request:

    https://github.com/apache/drill/pull/756#discussion_r102799356
  
    --- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/FragmentWrapper.java ---
    @@ -49,58 +51,135 @@ public String getId() {
         return String.format("fragment-%s", major.getMajorFragmentId());
       }
     
    -  public static final String[] FRAGMENT_OVERVIEW_COLUMNS = {"Major Fragment", "Minor Fragments Reporting",
    -    "First Start", "Last Start", "First End", "Last End", "Min Runtime", "Avg Runtime", "Max Runtime", "Last Update",
    -    "Last Progress", "Max Peak Memory"};
    +  public static final String[] ACTIVE_FRAGMENT_OVERVIEW_COLUMNS = {"Major Fragment", "Minor Fragments Reporting",
    +    "First Start", "Last Start", "First End", "Last End", "Min Runtime", "Avg Runtime", "Max Runtime", "% Busy",
    +    "Last Update", "Last Progress", "Max Peak Memory"};
    +
    +  public static final String[] ACTIVE_FRAGMENT_OVERVIEW_COLUMNS_TOOLTIP = {null, "# Minor Fragments Spawned",
    +    null, null, null, null, "Shortest duration of a fragment", "Avg duration of a fragment", "Longest duration of a fragment", "%time Fragments were Busy",
    +    "Last time a running fragment's status was updated", "Last time we heard from a running fragment", null};
     
       // Not including Major Fragment ID and Minor Fragments Reporting
    -  public static final int NUM_NULLABLE_OVERVIEW_COLUMNS = FRAGMENT_OVERVIEW_COLUMNS.length - 2;
    +  public static final int NUM_NULLABLE_ACTIVE_OVERVIEW_COLUMNS = ACTIVE_FRAGMENT_OVERVIEW_COLUMNS.length - 2;
     
       public void addSummary(TableBuilder tb) {
         // Use only minor fragments that have complete profiles
         // Complete iff the fragment profile has at least one operator profile, and start and end times.
         final List<MinorFragmentProfile> complete = new ArrayList<>(
           Collections2.filter(major.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes));
     
    -    tb.appendCell(new OperatorPathBuilder().setMajor(major).build(), null);
    -    tb.appendCell(complete.size() + " / " + major.getMinorFragmentProfileCount(), null);
    +    tb.appendCell(new OperatorPathBuilder().setMajor(major).build(), null, null);
    +    tb.appendCell(complete.size() + " / " + major.getMinorFragmentProfileCount(), null, null);
     
         // If there are no stats to aggregate, create an empty row
         if (complete.size() < 1) {
    -      tb.appendRepeated("", null, NUM_NULLABLE_OVERVIEW_COLUMNS);
    +      tb.appendRepeated("", null, NUM_NULLABLE_ACTIVE_OVERVIEW_COLUMNS, null);
           return;
         }
     
         final MinorFragmentProfile firstStart = Collections.min(complete, Comparators.startTime);
         final MinorFragmentProfile lastStart = Collections.max(complete, Comparators.startTime);
    -    tb.appendMillis(firstStart.getStartTime() - start, null);
    -    tb.appendMillis(lastStart.getStartTime() - start, null);
    +    tb.appendMillis(firstStart.getStartTime() - start, null, null);
    +    tb.appendMillis(lastStart.getStartTime() - start, null, null);
     
         final MinorFragmentProfile firstEnd = Collections.min(complete, Comparators.endTime);
         final MinorFragmentProfile lastEnd = Collections.max(complete, Comparators.endTime);
    -    tb.appendMillis(firstEnd.getEndTime() - start, null);
    -    tb.appendMillis(lastEnd.getEndTime() - start, null);
    +    tb.appendMillis(firstEnd.getEndTime() - start, null, null);
    +    tb.appendMillis(lastEnd.getEndTime() - start, null, null);
     
    -    long total = 0;
    +    long totalDuration = 0L;
    +    double totalProcessInMillis = 0.0d;
    +    double totalWaitInMillis = 0.0d;
         for (final MinorFragmentProfile p : complete) {
    -      total += p.getEndTime() - p.getStartTime();
    +      totalDuration += p.getEndTime() - p.getStartTime();
    +      //Capture Busy & Wait Time
    +      List<OperatorProfile> opProfileList = p.getOperatorProfileList();
    +      for (OperatorProfile operatorProfile : opProfileList) {
    +        totalProcessInMillis += operatorProfile.getProcessNanos()/1E6;
    +        totalWaitInMillis += operatorProfile.getWaitNanos()/1E6;
    +      }
         }
     
         final MinorFragmentProfile shortRun = Collections.min(complete, Comparators.runTime);
         final MinorFragmentProfile longRun = Collections.max(complete, Comparators.runTime);
    -    tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime(), null);
    -    tb.appendMillis(total / complete.size(), null);
    -    tb.appendMillis(longRun.getEndTime() - longRun.getStartTime(), null);
    +    tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime(), null, null);
    +    tb.appendMillis(totalDuration / complete.size(), null, null);
    +    tb.appendMillis(longRun.getEndTime() - longRun.getStartTime(), null, null);
    +
    +    tb.appendPercent(totalProcessInMillis / (totalProcessInMillis + totalWaitInMillis), null,
    +      String.format("&#8721;Busy: %,.2fs + &#8721;Wait: %,.2fs", totalProcessInMillis/1E3, totalWaitInMillis/1E3));
    --- End diff --
    
    Comment: #8721 is the summation sign.


> Publish Operator and MajorFragment Stats in Profile page
> --------------------------------------------------------
>
>                 Key: DRILL-5195
>                 URL: https://issues.apache.org/jira/browse/DRILL-5195
>             Project: Apache Drill
>          Issue Type: Improvement
>          Components: Web Server
>    Affects Versions: 1.9.0
>            Reporter: Kunal Khatua
>            Assignee: Kunal Khatua
>         Attachments: dbit_complete.png, dbit_inflight.png, dbit_opOverview.png
>
>
> Currently, we show runtimes for major fragments, and min,max,avg times for setup, processing and waiting for various operators.
> It would be worthwhile to have additional stats for the following:
> MajorFragment
>   %Busy - % of the active time for all the minor fragments within each major fragment that they were busy. 
> Operator Profile
>   %Busy - % of the active time for all the fragments within each operator that they were busy. 
>   Records - Total number of records propagated out by that operator.


--
This message was sent by Atlassian JIRA
(v6.3.15#6346)