chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r781603 - in /hadoop/chukwa/trunk/src: java/org/apache/hadoop/chukwa/hicc/ web/hicc/descriptors/ web/hicc/js/ web/hicc/jsp/
Date Wed, 03 Jun 2009 22:10:55 GMT
Author: eyang
Date: Wed Jun  3 22:10:54 2009
New Revision: 781603

URL: http://svn.apache.org/viewvc?rev=781603&view=rev
Log:
CHUKWA-274. Improve rate calculation for accumulated values for visualization. (Eric Yang)

Modified:
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/DatasetMapper.java
    hadoop/chukwa/trunk/src/web/hicc/descriptors/dfs_datanode_counter.descriptor
    hadoop/chukwa/trunk/src/web/hicc/descriptors/dfs_datanode_rate.descriptor
    hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_jvm_counter.descriptor
    hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_jvm_rate.descriptor
    hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_mapred.descriptor
    hadoop/chukwa/trunk/src/web/hicc/js/flot.extend.js
    hadoop/chukwa/trunk/src/web/hicc/jsp/single-series-chart-javascript.jsp

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/DatasetMapper.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/DatasetMapper.java?rev=781603&r1=781602&r2=781603&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/DatasetMapper.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/DatasetMapper.java Wed Jun
 3 22:10:54 2009
@@ -60,6 +60,8 @@
     labels.clear();
     double max = 0.0;
     int labelsCount = 0;
+    long timeWindowSize=0;
+    long previousTime=0;
     try {
       conn = org.apache.hadoop.chukwa.util.DriverManagerUtil.getConnection(jdbc);
       stmt = conn.prepareStatement(query);
@@ -86,6 +88,14 @@
           String label = "";
           if (rmeta.getColumnType(1) == java.sql.Types.TIMESTAMP) {
             long time = rs.getTimestamp(1).getTime();
+            if(time==previousTime) {
+            } else if(timeWindowSize==0) {
+              timeWindowSize=1;
+              previousTime=time;
+            } else {
+              timeWindowSize=(time-previousTime)/60000;
+              previousTime=time;
+            }
             label = "" + time;
           } else {
             label = rs.getString(1);
@@ -108,7 +118,10 @@
                 double current = rs.getDouble(j);
                 double tmp = 0L;
                 if (data.size() > 1) {
-                  tmp = current - previousHash.get(item).doubleValue();
+                  tmp = (current - previousHash.get(item).doubleValue())/timeWindowSize;
+                  if(tmp==Double.NEGATIVE_INFINITY || tmp==Double.POSITIVE_INFINITY) {
+                    tmp = Double.NaN;
+                  }
                 } else {
                   tmp = 0;
                 }
@@ -144,7 +157,10 @@
               if (calculateSlope) {
                 double tmp = current;
                 if (data.size() > 1) {
-                  tmp = tmp - previousArray[j];
+                  tmp = (tmp - previousArray[j])/timeWindowSize;
+                  if(tmp==Double.NEGATIVE_INFINITY || tmp==Double.POSITIVE_INFINITY) {
+                    tmp = Double.NaN;
+                  }
                 } else {
                   tmp = 0.0;
                 }

Modified: hadoop/chukwa/trunk/src/web/hicc/descriptors/dfs_datanode_counter.descriptor
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/web/hicc/descriptors/dfs_datanode_counter.descriptor?rev=781603&r1=781602&r2=781603&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/web/hicc/descriptors/dfs_datanode_counter.descriptor (original)
+++ hadoop/chukwa/trunk/src/web/hicc/descriptors/dfs_datanode_counter.descriptor Wed Jun 
3 22:10:54 2009
@@ -9,7 +9,6 @@
 "refresh":"15",
 "parameters":[
 {"name":"table","type":"string","value":"dfs_datanode","edit":"0"},
-{"name":"normalize_time","type":"string","value":"true","edit":"0"},
 {"name":"find_slope","type":"string","value":"true","edit":"0"},
 {"name":"group_items","type":"string","value":"hosts","edit":"0"},
 {"name":"group","type":"string","value":"host","edit":"0"},

Modified: hadoop/chukwa/trunk/src/web/hicc/descriptors/dfs_datanode_rate.descriptor
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/web/hicc/descriptors/dfs_datanode_rate.descriptor?rev=781603&r1=781602&r2=781603&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/web/hicc/descriptors/dfs_datanode_rate.descriptor (original)
+++ hadoop/chukwa/trunk/src/web/hicc/descriptors/dfs_datanode_rate.descriptor Wed Jun  3 22:10:54
2009
@@ -9,7 +9,6 @@
 "refresh":"15",
 "parameters":[
 {"name":"table","type":"string","value":"dfs_datanode","edit":"0"},
-{"name":"normalize_time","type":"string","value":"true","edit":"0"},
 {"name":"group_items","type":"string","value":"hosts","edit":"0"},
 {"name":"group","type":"string","value":"host","edit":"0"},
 {"name":"period","type":"custom","control":"period_control","value":"","label":"Period"},

Modified: hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_jvm_counter.descriptor
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_jvm_counter.descriptor?rev=781603&r1=781602&r2=781603&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_jvm_counter.descriptor (original)
+++ hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_jvm_counter.descriptor Wed Jun  3
22:10:54 2009
@@ -14,7 +14,7 @@
 {"name":"group","type":"string","value":"host","edit":"0"},
 {"name":"group_items","type":"string","value":"hosts","edit":"0"},
 {"name":"period","type":"custom","control":"period_control","value":"","label":"Period"},
-{"name":"metric","type":"select","value":"gc_timemillis","label":"Metric","options":[
+{"name":"metric","type":"select_multiple","value":"gc_timemillis","label":"Metric","options":[
 {"label":"gc_timemillis","value":"gc_timemillis"},
 {"label":"gc_count","value":"gc_count"},
 {"label":"log_error","value":"log_error"},
@@ -22,6 +22,13 @@
 {"label":"log_info","value":"log_info"},
 {"label":"log_warn","value":"log_warn"}
 ]},
+{"name":"match","type":"select_multiple","value":"datanode","label":"Process Type","options":[
+{"label":"datanode","value":"process_name='datanode'"},
+{"label":"jobtracker","value":"process_name='jobtracker'"},
+{"label":"namenode","value":"process_name='namenode'"},
+{"label":"shuffle","value":"process_name='shuffle'"},
+{"label":"tasktracker","value":"process_name='tasktracker'"},
+]},
 {"name":"width","type":"select","value":"300","label":"Width","options":[
 {"label":"300","value":"300"},
 {"label":"400","value":"400"},

Modified: hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_jvm_rate.descriptor
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_jvm_rate.descriptor?rev=781603&r1=781602&r2=781603&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_jvm_rate.descriptor (original)
+++ hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_jvm_rate.descriptor Wed Jun  3 22:10:54
2009
@@ -9,12 +9,11 @@
 "refresh":"15",
 "parameters":[
 {"name":"table","type":"string","value":"hadoop_jvm","edit":"0"},
-{"name":"normalize_time","type":"string","value":"true","edit":"0"},
 {"name":"group_override","type":"string","value":"concat(host,' ',process_name) as process","edit":"0"},
 {"name":"group","type":"string","value":"host","edit":"0"},
 {"name":"group_items","type":"string","value":"hosts","edit":"0"},
 {"name":"period","type":"custom","control":"period_control","value":"","label":"Period"},
-{"name":"metric","type":"select","value":"threads_blocked","label":"Metric","options":[
+{"name":"metric","type":"select_multiple","value":"threads_blocked","label":"Metric","options":[
 {"label":"mem_heap_committed_m","value":"mem_heap_committed_m"},
 {"label":"mem_heap_used_m","value":"mem_heap_used_m"},
 {"label":"mem_non_heap_committed_m","value":"mem_non_heap_committed_m"},
@@ -26,6 +25,13 @@
 {"label":"threads_timed_waiting","value":"threads_timed_waiting"},
 {"label":"threads_waiting","value":"threads_waiting"},
 ]},
+{"name":"match","type":"select_multiple","value":"datanode","label":"Process Type","options":[
+{"label":"datanode","value":"process_name='datanode'"},
+{"label":"jobtracker","value":"process_name='jobtracker'"},
+{"label":"namenode","value":"process_name='namenode'"},
+{"label":"shuffle","value":"process_name='shuffle'"},
+{"label":"tasktracker","value":"process_name='tasktracker'"},
+]},
 {"name":"width","type":"select","value":"300","label":"Width","options":[
 {"label":"300","value":"300"},
 {"label":"400","value":"400"},

Modified: hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_mapred.descriptor
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_mapred.descriptor?rev=781603&r1=781602&r2=781603&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_mapred.descriptor (original)
+++ hadoop/chukwa/trunk/src/web/hicc/descriptors/hadoop_mapred.descriptor Wed Jun  3 22:10:54
2009
@@ -11,7 +11,7 @@
 {"name":"find_slope","type":"string","value":"true","edit":"0"},
 {"name":"table","type":"string","value":"hadoop_mapred","edit":"0"},
 {"name":"period","type":"custom","control":"period_control","value":"","label":"Period"},
-{"name":"metric","type":"select_multiple","value":"heartbeat_avg_time","label":"Metric","options":[
+{"name":"metric","type":"select_multiple","value":"jobs_completed","label":"Metric","options":[
 {"label":"jobs_completed","value":"jobs_completed"},
 {"label":"jobs_submitted","value":"jobs_submitted"},
 {"label":"maps_completed","value":"maps_completed"},

Modified: hadoop/chukwa/trunk/src/web/hicc/js/flot.extend.js
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/web/hicc/js/flot.extend.js?rev=781603&r1=781602&r2=781603&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/web/hicc/js/flot.extend.js (original)
+++ hadoop/chukwa/trunk/src/web/hicc/js/flot.extend.js Wed Jun  3 22:10:54 2009
@@ -14,8 +14,12 @@
   var last=0;
   var sum_sqr=0.0;
   for (i=0; i<data.length;i++) {
-    x=data[i][0];
-    y=data[i][1];
+    try {
+      x=data[i][0];
+      y=data[i][1];
+    } catch(nullException) {
+      continue;
+    }
     if (bound!=null) {
       if (!((x >= bound.xmin) && (x <=bound.xmax) &&
 	  (y >= bound.ymin) && (y <=bound.ymax))) {

Modified: hadoop/chukwa/trunk/src/web/hicc/jsp/single-series-chart-javascript.jsp
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/web/hicc/jsp/single-series-chart-javascript.jsp?rev=781603&r1=781602&r2=781603&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/web/hicc/jsp/single-series-chart-javascript.jsp (original)
+++ hadoop/chukwa/trunk/src/web/hicc/jsp/single-series-chart-javascript.jsp Wed Jun  3 22:10:54
2009
@@ -51,7 +51,7 @@
     if(cluster==null) {
          cluster="demo";
     }
-    String match=xf.getParameter("match");
+    String[] match=xf.getParameterValues("match");
     String group = xf.getParameter("group");
     ClusterConfig cc = new ClusterConfig();
     String jdbc = cc.getURL(cluster);
@@ -78,6 +78,50 @@
             }
         }
     }
+    StringBuffer condition=new StringBuffer();
+    if(request.getParameter("group_items")!=null) {
+        if(session.getAttribute(xf.getParameter("group_items"))==null) {
+            session.setAttribute(xf.getParameter("group_items"),"");
+        }
+        int counter = 0;
+        String[] group_items = ((String)session.getAttribute(xf.getParameter("group_items"))).split(",");
+        if(group_items!=null) {
+            StringBuilder matchBuilder = new StringBuilder();
+            for(String item : group_items) {
+                if(counter!=0) {
+                    matchBuilder.append(" or ");
+                } else {
+                    matchBuilder.append("(");
+                }
+                matchBuilder.append(group);
+                matchBuilder.append(" = ?");
+                parms.add(item);
+                counter++;
+            }
+            if(counter!=0) {
+                matchBuilder.append(")");
+                if(condition.toString().intern()!="".intern()) {
+                    condition.append(" and ");
+                }
+                condition.append(matchBuilder.toString());
+            }
+        }
+    }
+    if(match!=null) {
+        if(condition.toString().intern()!="".intern()) {
+            condition.append(" and ");
+        }
+        condition.append("(");
+        boolean first=true;
+        for(String item : match) {
+            if(!first) {
+                condition.append(" or ");
+                first=false;
+            }
+            condition.append(item.replaceAll(","," or ").replaceAll("%27","'").replaceAll(";",""));
+        }
+        condition.append(")");
+    }
     String random = xf.getParameter("_s");
     TimeHandler time = new TimeHandler(request, (String)session.getAttribute("time_zone"));
     startS = time.getStartTimeText();
@@ -102,37 +146,12 @@
        Connection conn = null;
        Statement stmt = null;
        ResultSet rs = null;
-       if(request.getParameter("group_items")!=null) {
-           if(session.getAttribute(xf.getParameter("group_items"))==null) {
-               session.setAttribute(xf.getParameter("group_items"),"");
-           }
-           int counter = 0;
-           String[] group_items = ((String)session.getAttribute(xf.getParameter("group_items"))).split(",");
-           if(group_items!=null) {
-               StringBuilder matchBuilder = new StringBuilder();
-               for(String item : group_items) {
-                   if(counter!=0) {
-                       matchBuilder.append("or");
-                   } else {
-                       matchBuilder.append("(");
-                   }
-                   matchBuilder.append(group);
-                   matchBuilder.append(" = ? ");
-                   parms.add(item);
-                   counter++;
-               }
-               if(counter!=0) {
-                   matchBuilder.append(")");
-                   match = matchBuilder.toString();
-               }
-           }
-       }
        String table = (String)xf.getParameter("table");
        if(table==null) {
            table = "cluster_system_metrics";
        }
        if(request.getParameter("group_override")!=null) {
-           group=xf.getParameter("group_override");
+           group=(xf.getParameter("group_override").replaceAll("%27","'").replaceAll(";",""));
        }
        String[] tables = null;
        DatabaseConfig dbc = new DatabaseConfig();
@@ -153,10 +172,8 @@
            q.append(" from ");
            q.append(tmpTable);
            q.append(" where ");
-           if(match!=null) {
-             q.append(match);
-           }
-           if(match!=null && match.intern()!="".intern()) {
+           if(condition.toString().intern()!="".intern()) {
+             q.append(condition.toString());
              q.append(" and ");
            }
            q.append(dateclause);



Mime
View raw message