drill-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From paul-rogers <...@git.apache.org>
Subject [GitHub] drill pull request #756: DRILL-5195: Publish Operator and MajorFragment Stat...
Date Thu, 23 Feb 2017 20:16:05 GMT
Github user paul-rogers commented on a diff in the pull request:

    https://github.com/apache/drill/pull/756#discussion_r102801678
  
    --- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/FragmentWrapper.java
---
    @@ -49,58 +51,135 @@ public String getId() {
         return String.format("fragment-%s", major.getMajorFragmentId());
       }
     
    -  public static final String[] FRAGMENT_OVERVIEW_COLUMNS = {"Major Fragment", "Minor
Fragments Reporting",
    -    "First Start", "Last Start", "First End", "Last End", "Min Runtime", "Avg Runtime",
"Max Runtime", "Last Update",
    -    "Last Progress", "Max Peak Memory"};
    +  public static final String[] ACTIVE_FRAGMENT_OVERVIEW_COLUMNS = {"Major Fragment",
"Minor Fragments Reporting",
    +    "First Start", "Last Start", "First End", "Last End", "Min Runtime", "Avg Runtime",
"Max Runtime", "% Busy",
    +    "Last Update", "Last Progress", "Max Peak Memory"};
    +
    +  public static final String[] ACTIVE_FRAGMENT_OVERVIEW_COLUMNS_TOOLTIP = {null, "# Minor
Fragments Spawned",
    +    null, null, null, null, "Shortest duration of a fragment", "Avg duration of a fragment",
"Longest duration of a fragment", "%time Fragments were Busy",
    +    "Last time a running fragment's status was updated", "Last time we heard from a running
fragment", null};
     
       // Not including Major Fragment ID and Minor Fragments Reporting
    -  public static final int NUM_NULLABLE_OVERVIEW_COLUMNS = FRAGMENT_OVERVIEW_COLUMNS.length
- 2;
    +  public static final int NUM_NULLABLE_ACTIVE_OVERVIEW_COLUMNS = ACTIVE_FRAGMENT_OVERVIEW_COLUMNS.length
- 2;
     
       public void addSummary(TableBuilder tb) {
         // Use only minor fragments that have complete profiles
         // Complete iff the fragment profile has at least one operator profile, and start
and end times.
         final List<MinorFragmentProfile> complete = new ArrayList<>(
           Collections2.filter(major.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes));
     
    -    tb.appendCell(new OperatorPathBuilder().setMajor(major).build(), null);
    -    tb.appendCell(complete.size() + " / " + major.getMinorFragmentProfileCount(), null);
    +    tb.appendCell(new OperatorPathBuilder().setMajor(major).build(), null, null);
    +    tb.appendCell(complete.size() + " / " + major.getMinorFragmentProfileCount(), null,
null);
     
         // If there are no stats to aggregate, create an empty row
         if (complete.size() < 1) {
    -      tb.appendRepeated("", null, NUM_NULLABLE_OVERVIEW_COLUMNS);
    +      tb.appendRepeated("", null, NUM_NULLABLE_ACTIVE_OVERVIEW_COLUMNS, null);
           return;
         }
     
         final MinorFragmentProfile firstStart = Collections.min(complete, Comparators.startTime);
         final MinorFragmentProfile lastStart = Collections.max(complete, Comparators.startTime);
    -    tb.appendMillis(firstStart.getStartTime() - start, null);
    -    tb.appendMillis(lastStart.getStartTime() - start, null);
    +    tb.appendMillis(firstStart.getStartTime() - start, null, null);
    +    tb.appendMillis(lastStart.getStartTime() - start, null, null);
     
         final MinorFragmentProfile firstEnd = Collections.min(complete, Comparators.endTime);
         final MinorFragmentProfile lastEnd = Collections.max(complete, Comparators.endTime);
    -    tb.appendMillis(firstEnd.getEndTime() - start, null);
    -    tb.appendMillis(lastEnd.getEndTime() - start, null);
    +    tb.appendMillis(firstEnd.getEndTime() - start, null, null);
    +    tb.appendMillis(lastEnd.getEndTime() - start, null, null);
     
    -    long total = 0;
    +    long totalDuration = 0L;
    +    double totalProcessInMillis = 0.0d;
    +    double totalWaitInMillis = 0.0d;
         for (final MinorFragmentProfile p : complete) {
    -      total += p.getEndTime() - p.getStartTime();
    +      totalDuration += p.getEndTime() - p.getStartTime();
    +      //Capture Busy & Wait Time
    +      List<OperatorProfile> opProfileList = p.getOperatorProfileList();
    +      for (OperatorProfile operatorProfile : opProfileList) {
    +        totalProcessInMillis += operatorProfile.getProcessNanos()/1E6;
    +        totalWaitInMillis += operatorProfile.getWaitNanos()/1E6;
    +      }
         }
     
         final MinorFragmentProfile shortRun = Collections.min(complete, Comparators.runTime);
         final MinorFragmentProfile longRun = Collections.max(complete, Comparators.runTime);
    -    tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime(), null);
    -    tb.appendMillis(total / complete.size(), null);
    -    tb.appendMillis(longRun.getEndTime() - longRun.getStartTime(), null);
    +    tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime(), null, null);
    +    tb.appendMillis(totalDuration / complete.size(), null, null);
    +    tb.appendMillis(longRun.getEndTime() - longRun.getStartTime(), null, null);
    +
    +    tb.appendPercent(totalProcessInMillis / (totalProcessInMillis + totalWaitInMillis),
null,
    +      String.format("&#8721;Busy: %,.2fs + &#8721;Wait: %,.2fs", totalProcessInMillis/1E3,
totalWaitInMillis/1E3));
     
         final MinorFragmentProfile lastUpdate = Collections.max(complete, Comparators.lastUpdate);
    -    tb.appendTime(lastUpdate.getLastUpdate(), null);
    +    tb.appendMillis(System.currentTimeMillis()-lastUpdate.getLastUpdate(), null, new
SimpleDateFormat().format(new Date(lastUpdate.getLastUpdate())));
     
         final MinorFragmentProfile lastProgress = Collections.max(complete, Comparators.lastProgress);
    -    tb.appendTime(lastProgress.getLastProgress(), null);
    +    tb.appendMillis(System.currentTimeMillis()-lastProgress.getLastProgress(), null,
new SimpleDateFormat().format(new Date(lastProgress.getLastProgress())));
    +
    +    // TODO(DRILL-3494): Names (maxMem, getMaxMemoryUsed) are misleading; the value is
peak memory allocated to fragment
    +    final MinorFragmentProfile maxMem = Collections.max(complete, Comparators.fragmentPeakMemory);
    +    tb.appendBytes(maxMem.getMaxMemoryUsed(), null, null);
    +  }
    +
    +  public static final String[] COMPLETED_FRAGMENT_OVERVIEW_COLUMNS = {"Major Fragment",
"Minor Fragments Reporting",
    +    "First Start", "Last Start", "First End", "Last End", "Min Runtime", "Avg Runtime",
"Max Runtime", "% Busy", "Max Peak Memory"};
    +
    +  public static final String[] COMPLETED_FRAGMENT_OVERVIEW_COLUMNS_TOOLTIP = {null, "#
Minor Fragments Spawned",
    +    null, null, null, null, "Shortest duration of a fragment", "Avg duration of a fragment",
"Longest duration of a fragment", "%time Fragments were Busy", null};
    +
    +  //Not including Major Fragment ID and Minor Fragments Reporting
    +  public static final int NUM_NULLABLE_COMPLETED_OVERVIEW_COLUMNS = COMPLETED_FRAGMENT_OVERVIEW_COLUMNS.length
- 2;
    +
    +  public void addFinalSummary(TableBuilder tb) {
    +
    +    // Use only minor fragments that have complete profiles
    +    // Complete iff the fragment profile has at least one operator profile, and start
and end times.
    +    final List<MinorFragmentProfile> complete = new ArrayList<>(
    +      Collections2.filter(major.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes));
    +
    +    tb.appendCell(new OperatorPathBuilder().setMajor(major).build(), null, null);
    +    tb.appendCell(complete.size() + " / " + major.getMinorFragmentProfileCount(), null,
null);
    +
    +    // If there are no stats to aggregate, create an empty row
    +    if (complete.size() < 1) {
    +      tb.appendRepeated("", null, NUM_NULLABLE_COMPLETED_OVERVIEW_COLUMNS, null);
    +      return;
    +    }
    +
    +    final MinorFragmentProfile firstStart = Collections.min(complete, Comparators.startTime);
    --- End diff --
    
    Does this handle the case in which the start or end times are not yet known? The query
will appear to run a long time if one fragment has not started (time stamp of 0) and another
has ended (time stamp of now)...


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message