drill-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (DRILL-5260) Refinements to new "Cluster Fixture" test framework
Date Fri, 24 Feb 2017 23:06:44 GMT

    [ https://issues.apache.org/jira/browse/DRILL-5260?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15883712#comment-15883712
] 

ASF GitHub Bot commented on DRILL-5260:
---------------------------------------

Github user sohami commented on a diff in the pull request:

    https://github.com/apache/drill/pull/753#discussion_r103052237
  
    --- Diff: exec/java-exec/src/test/java/org/apache/drill/test/ProfileParser.java ---
    @@ -138,9 +414,208 @@ public long getMetric(int id) {
         }
       }
     
    -  public Map<Integer,OpInfo> getOpInfo( ) {
    +  /**
    +   * Information about an operator definition: the plan-time information
    +   * that appears in the plan portion of the profile. Also holds the
    +   * "actuals" from the minor fragment portion of the profile.
    +   * Allows integrating the "planned" vs. "actual" performance of the
    +   * query.
    +   */
    +
    +  public static class OpDefInfo {
    +    public String opName;
    +    public boolean isInferred;
    +    public int majorId;
    +    public int stepId;
    +    public String args;
    +    public List<FieldDef> columns;
    +    public int globalLevel;
    +    public int localLevel;
    +    public int id;
    +    public int branchId;
    +    public boolean isBranchRoot;
    +    public double estMemoryCost;
    +    public double estNetCost;
    +    public double estIOCost;
    +    public double estCpuCost;
    +    public double estRowCost;
    +    public double estRows;
    +    public String name;
    +    public long actualMemory;
    +    public int actualBatches;
    +    public long actualRows;
    +    public OpDefInfo inferredParent;
    +    public List<OperatorProfile> opExecs = new ArrayList<>( );
    +    public List<OpDefInfo> children = new ArrayList<>( );
    +
    +    // 00-00    Screen : rowType = RecordType(VARCHAR(10) Year, VARCHAR(65536) Month,
VARCHAR(100) Devices, VARCHAR(100) Tier, VARCHAR(100) LOB, CHAR(10) Gateway, BIGINT Day, BIGINT
Hour, INTEGER Week, VARCHAR(100) Week_end_date, BIGINT Usage_Cnt): \
    +    // rowcount = 100.0, cumulative cost = {7.42124276972414E9 rows, 7.663067406383167E10
cpu, 0.0 io, 2.24645048816E10 network, 2.692766612982188E8 memory}, id = 129302
    +    //
    +    // 00-01      Project(Year=[$0], Month=[$1], Devices=[$2], Tier=[$3], LOB=[$4], Gateway=[$5],
Day=[$6], Hour=[$7], Week=[$8], Week_end_date=[$9], Usage_Cnt=[$10]) :
    +    // rowType = RecordType(VARCHAR(10) Year, VARCHAR(65536) Month, VARCHAR(100) Devices,
VARCHAR(100) Tier, VARCHAR(100) LOB, CHAR(10) Gateway, BIGINT Day, BIGINT Hour, INTEGER Week,
VARCHAR(100) Week_end_date, BIGINT Usage_Cnt): rowcount = 100.0, cumulative cost = {7.42124275972414E9
rows, 7.663067405383167E10 cpu, 0.0 io, 2.24645048816E10 network, 2.692766612982188E8 memory},
id = 129301
    +
    +    public OpDefInfo(String plan) {
    +      Pattern p = Pattern.compile( "^(\\d+)-(\\d+)(\\s+)(\\w+)(?:\\((.*)\\))?\\s*:\\s*(.*)$"
);
    +      Matcher m = p.matcher(plan);
    +      if (!m.matches()) {
    +        throw new IllegalStateException( "Could not parse plan: " + plan );
    +      }
    +      majorId = Integer.parseInt(m.group(1));
    +      stepId = Integer.parseInt(m.group(2));
    +      name = m.group(4);
    +      args = m.group(5);
    +      String tail = m.group(6);
    +      String indent = m.group(3);
    +      globalLevel = (indent.length() - 4) / 2;
    +
    +      p = Pattern.compile("rowType = RecordType\\((.*)\\): (rowcount .*)");
    +      m = p.matcher(tail);
    +      if ( m.matches() ) {
    +        columns = parseCols(m.group(1));
    +        tail = m.group(2);
    +      }
    +
    +      p = Pattern.compile( "rowcount = ([\\d.E]+), cumulative cost = \\{([\\d.E]+) rows,
([\\d.E]+) cpu, ([\\d.E]+) io, ([\\d.E]+) network, ([\\d.E]+) memory\\}, id = (\\d+)");
    +      m = p.matcher(tail);
    +      if (! m.matches()) {
    +        throw new IllegalStateException("Could not parse costs: " + tail );
    +      }
    +      estRows = Double.parseDouble(m.group(1));
    +      estRowCost = Double.parseDouble(m.group(2));
    +      estCpuCost = Double.parseDouble(m.group(3));
    +      estIOCost = Double.parseDouble(m.group(4));
    +      estNetCost = Double.parseDouble(m.group(5));
    +      estMemoryCost = Double.parseDouble(m.group(6));
    +      id = Integer.parseInt(m.group(7));
    +    }
    +
    +    public void printTree(String indent) {
    +      new TreePrinter().visit(this);
    +    }
    +
    +    public OpDefInfo(int major, int id) {
    +      majorId = major;
    +      stepId = id;
    +    }
    +
    +    @Override
    +    public String toString() {
    +      String head = "[OpDefInfo " + majorId + "-" + stepId + ": " + name;
    +      if ( isInferred ) {
    +        head += " (" + opName + ")";
    +      }
    +      return head + "]";
    +    }
    +  }
    +
    +  /**
    +   * Visit a tree of operator definitions to support printing,
    +   * analysis and other tasks.
    +   */
    +
    +  public static class TreeVisitor
    +  {
    +    public void visit(OpDefInfo root) {
    +      visit(root, 0);
    +    }
    +    public void visit(OpDefInfo node, int indent) {
    +      visitOp( node, indent );
    +      if (node.children.isEmpty()) {
    +        return;
    +      }
    +      if ( node.children.size() == 1) {
    +        visit(node.children.get(0), indent);
    +        return;
    +      }
    +      indent++;
    +      int i = 0;
    +      for (OpDefInfo child : node.children) {
    +        visitSubtree(node, i++, indent);
    +        visit(child, indent+1);
    +      }
    +    }
    +
    +    protected void visitOp(OpDefInfo node, int indent) {
    +    }
    +
    +    protected void visitSubtree(OpDefInfo node, int i, int indent) {
    +    }
    +
    +    public String indentString(int indent, String pad) {
    +      StringBuilder buf = new StringBuilder();
    +      for (int i = 0; i < indent; i++) {
    +        buf.append( pad );
    +      }
    +      return buf.toString();
    +    }
    +
    +    public String indentString(int indent) {
    +      return indentString(indent, "  ");
    +    }
    +
    +    public String subtreeLabel(OpDefInfo node, int branch) {
    +      if (node.name.equals("HashJoin")) {
    +        return (branch == 0) ? "Probe" : "Build";
    +      } else {
    +        return "Input " + (branch + 1);
    +      }
    +    }
    +  }
    +
    +  /**
    +   * Print the operator tree for analysis.
    +   */
    +
    +  public static class TreePrinter extends TreeVisitor
    +  {
    +    @Override
    +    protected void visitOp(OpDefInfo node, int indent) {
    +      System.out.print( indentString(indent) );
    +      System.out.println( node.toString() );
    +    }
    +
    +    @Override
    +    protected void visitSubtree(OpDefInfo node, int i, int indent) {
    +      System.out.print( indentString(indent) );
    +      System.out.println(subtreeLabel(node, i));
    +    }
    +  }
    +
    +  /**
    +   * Print out the tree showing a comparison of estimated vs.
    +   * actual costs.
    +   */
    +
    +  public static class CostPrinter extends TreeVisitor
    +  {
    +    @Override
    +    protected void visitOp(OpDefInfo node, int indentLevel) {
    +      System.out.print(String.format("%02d-%02d ", node.majorId, node.stepId));
    +      String indent = indentString(indentLevel, ". ");
    +      System.out.print( indent + node.name );
    +      if (node.opName != null) {
    +        System.out.print( " (" + node.opName + ")" );
    +      }
    +      System.out.println( );
    +      indent = indentString(15);
    +      System.out.print( indent );
    +      System.out.println(String.format("  Estimate: %,15.0f rows, %,7.0f MB",
    +                         node.estRows, node.estMemoryCost / 1024 / 1024) );
    +      System.out.print( indent );
    +      System.out.println(String.format("  Actual:   %,15d rows, %,7d MB",
    +                         node.actualRows, node.actualMemory / 1024 / 1024));
    +    }
    +
    +    @Override
    +    protected void visitSubtree(OpDefInfo node, int i, int indent) {
    +      System.out.print( indentString(indent) + "      " );
    +      System.out.println(subtreeLabel(node, i));
    +    }
    +  }
    +
    +  public Map<Integer,OperatorProfile> getOpInfo( ) {
         Map<Integer,String> ops = getOperators( );
    --- End diff --
    
    Thanks!


> Refinements to new "Cluster Fixture" test framework
> ---------------------------------------------------
>
>                 Key: DRILL-5260
>                 URL: https://issues.apache.org/jira/browse/DRILL-5260
>             Project: Apache Drill
>          Issue Type: Improvement
>    Affects Versions: 1.10
>            Reporter: Paul Rogers
>            Assignee: Paul Rogers
>            Priority: Minor
>             Fix For: 1.10
>
>
> Roll-up of a number of enhancements to the cluster fixture framework.
> * Config option to suppress printing of CSV and other output. (Allows printing for single
tests, not printing when running from Maven.)
> * Parsing of query profiles to extract plan and run time information.
> * Fix bug in log fixture when enabling logging for a package.
> * Improved ZK support.
> * Set up the new CTTAS default temporary workspace for tests.
> * Revise TestDrillbitResiliance to use the new framework.
> * Revise TestWindowFrame to to use the new framework.
> * Revise TestMergeJoinWithSchemaChanges to use the new framework.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Mime
View raw message