chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r1050592 - in /incubator/chukwa/trunk: ./ conf/ src/docs/src/documentation/content/xdocs/ src/java/org/apache/hadoop/chukwa/datastore/ src/java/org/apache/hadoop/chukwa/hicc/ src/java/org/apache/hadoop/chukwa/util/ src/web/hicc/jsp/
Date Sat, 18 Dec 2010 07:35:00 GMT
Author: eyang
Date: Sat Dec 18 07:34:59 2010
New Revision: 1050592

URL: http://svn.apache.org/viewvc?rev=1050592&view=rev
Log:
CHUKWA-554. Remove jdbc.conf dependency from Chukwa. (Eric Yang)

Removed:
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/WatchDog.java
    incubator/chukwa/trunk/src/web/hicc/jsp/get_user_list.jsp
    incubator/chukwa/trunk/src/web/hicc/jsp/hod_job.jsp
    incubator/chukwa/trunk/src/web/hicc/jsp/single-series-chart-javascript.jsp
Modified:
    incubator/chukwa/trunk/CHANGES.txt
    incubator/chukwa/trunk/conf/README
    incubator/chukwa/trunk/src/docs/src/documentation/content/xdocs/admin.xml
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/ClusterConfig.java
    incubator/chukwa/trunk/src/web/hicc/jsp/cluster_selector.jsp
    incubator/chukwa/trunk/src/web/hicc/jsp/event2_viewer.jsp

Modified: incubator/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/CHANGES.txt?rev=1050592&r1=1050591&r2=1050592&view=diff
==============================================================================
--- incubator/chukwa/trunk/CHANGES.txt (original)
+++ incubator/chukwa/trunk/CHANGES.txt Sat Dec 18 07:34:59 2010
@@ -28,6 +28,8 @@ Trunk (unreleased changes)
 
   IMPROVEMENTS
 
+    CHUKWA-554. Remove jdbc.conf dependency from Chukwa. (Eric Yang)
+
     CHUKWA-562. Added RegExDirTailingAdaptor. (Deepak Deshpande via Eric Yang)
 
     CHUKWA-474. Replaced JSON library with JSON-Simple. (Eric Yang)

Modified: incubator/chukwa/trunk/conf/README
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/conf/README?rev=1050592&r1=1050591&r2=1050592&view=diff
==============================================================================
--- incubator/chukwa/trunk/conf/README (original)
+++ incubator/chukwa/trunk/conf/README Sat Dec 18 07:34:59 2010
@@ -29,11 +29,6 @@ chukwa-collector-conf.xml.template 
 > The name node address, used by collectors to write data to HDFS 
 > eg: write.hdf.filesystem = hdfs://localhost:9000/
 --------------------------------------------
-jdbc.conf.template
-> Modify? yes     			
-> URL for the MySQL database 
-> eg: jdbc:mysql://localhost:3306?user-root
----------------------------------------------
 nagios.properties.template
 > Modify? yes, optional   	
 > Nagio server that receives NSCA alert 
@@ -127,4 +122,4 @@ hadoop-metrics-log4j.properties
 hadoop-metrics.properties.template
 > Modify? no	
 > Sample metrics for Chukwa cluster.  
-> Sample metrics for source nodes, if set up as a Hadoop cluster.
\ No newline at end of file
+> Sample metrics for source nodes, if set up as a Hadoop cluster.

Modified: incubator/chukwa/trunk/src/docs/src/documentation/content/xdocs/admin.xml
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/docs/src/documentation/content/xdocs/admin.xml?rev=1050592&r1=1050591&r2=1050592&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/docs/src/documentation/content/xdocs/admin.xml (original)
+++ incubator/chukwa/trunk/src/docs/src/documentation/content/xdocs/admin.xml Sat Dec 18 07:34:59
2010
@@ -329,56 +329,6 @@ source /path/to/chukwa/conf/database_upg
 <source>CHUKWA&#95;HOME/bin/downSampling.sh --config &#60;path to chukwa conf&#62;
-n add </source>
 </section>
 
-
-<section>
-<title>Set Up the Database </title>
-<p>Set up and configure the MySQL database.</p>
-
-<section>
-<title>Install MySQL</title>
-
-<p>Download MySQL 5.1 from the <a href="http://dev.mysql.com/downloads/mysql/5.1.html#downloads">MySQL
site</a>. </p>
-<source>
-tar fxvz mysql-&#42;.tar.gz -C $CHUKWA&#95;HOME/opt
-cd $CHUKWA&#95;HOME/opt/mysql-&#42;
-</source>
-
-<p>
-Configure and then copy the my.cnf file to the CHUKWA_HOME/opt/mysql-* directory:
-</p>
-<source>
-./scripts/mysql_install_db
-./bin/mysqld_safe&#38;
-./bin/mysqladmin -u root create &#60;clustername&#62;
-./bin/mysql -u root &#60;clustername&#62; &#60; $CHUKWA&#95;HOME/conf/database_create_tables.sql
-</source>
-
-<p>Edit the CHUKWA_HOME/conf/jdbc.conf configuration file. </p>
-<p>Set the clustername to the MYSQL root URL:</p>
-<source>
-&#60;clustername&#62;&#61;jdbc:mysql://localhost:3306/&#60;clustername&#62;?user&#61;root
-</source>
-
-<p>Download the MySQL Connector/J 5.1 from the  <a href="http://dev.mysql.com/downloads/connector/j/5.1.html">MySQL
site</a>, 
-and place the jar file in $CHUKWA_HOME/lib.</p>
-</section>
-
-<section>
-<title>Set Up MySQL for Replication</title>
-<p>Start the MySQL shell:</p>
-<source>
-mysql -u root -p
-Enter password:
-</source>
-<p>From the MySQL shell, enter these commands (replace &#60;username&#62; and
&#60;password&#62; with actual values):</p>
-<source>
-GRANT REPLICATION SLAVE ON &#42;.&#42; TO &#39;&#60;username&#62;&#39;&#64;&#39;&#37;&#39;
IDENTIFIED BY &#39;&#60;password&#62;&#39;;
-FLUSH PRIVILEGES; 
-</source>
-</section>
-
-</section>
-
 <!-- 
 <section>
 <title>Validate the Chukwa Processes </title>

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java?rev=1050592&r1=1050591&r2=1050592&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java
(original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java
Sat Dec 18 07:34:59 2010
@@ -7,6 +7,8 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.hadoop.chukwa.datacollection.writer.hbase.HBaseWriter;
 import org.apache.hadoop.chukwa.hicc.bean.Series;
@@ -161,4 +163,32 @@ public class ChukwaHBaseStore {
   public static Set<String> getHostnames(String cluster, long startTime, long endTime)
{
     return getRowNames("SystemMetrics","system:csource", startTime, endTime);
   }
+  
+  public static Set<String> getClusterNames(long startTime, long endTime) {
+    String tableName = "SystemMetrics";
+    String column = "system:ctags";
+    Set<String> clusters = new HashSet<String>();
+    HTable table = pool.getTable(tableName);
+    Pattern p = Pattern.compile("\\s*cluster=\"(.*?)\"");
+    try {
+      Scan scan = new Scan();
+      scan.addColumn(column.getBytes());
+      scan.setTimeRange(startTime, endTime);
+      ResultScanner results = table.getScanner(scan);
+      Iterator<Result> it = results.iterator();
+      while(it.hasNext()) {
+        Result result = it.next();
+        String buffer = new String(result.getValue(column.getBytes()));
+        Matcher m = p.matcher(buffer);
+        if(m.matches()) {
+          clusters.add(m.group(1));
+        }
+      }
+      results.close();
+      table.close();
+    } catch(Exception e) {
+      log.error(ExceptionUtil.getStackTrace(e));
+    }
+    return clusters;
+  }
 }

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/ClusterConfig.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/ClusterConfig.java?rev=1050592&r1=1050591&r2=1050592&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/ClusterConfig.java (original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/ClusterConfig.java Sat Dec
18 07:34:59 2010
@@ -22,8 +22,10 @@ package org.apache.hadoop.chukwa.hicc;
 import java.io.*;
 import java.util.*;
 
+import org.apache.hadoop.chukwa.datastore.ChukwaHBaseStore;
+
 public class ClusterConfig {
-  public static HashMap<String, String> clusterMap = new HashMap<String, String>();
+  public static Set<String> clusterMap = null;
   private String path = System.getenv("CHUKWA_CONF_DIR") + File.separator;
 
   static public String getContents(File aFile) {
@@ -56,23 +58,15 @@ public class ClusterConfig {
   }
 
   public ClusterConfig() {
-    File cc = new File(path + "jdbc.conf");
-    String buffer = getContents(cc);
-    String[] lines = buffer.split("\n");
-    for (String line : lines) {
-      String[] data = line.split("=", 2);
-      clusterMap.put(data[0], data[1]);
+    long end = System.currentTimeMillis();
+    long start = end - 3600000L;
+    if(clusterMap==null) {
+      clusterMap = ChukwaHBaseStore.getClusterNames(start, end);
     }
   }
 
-  public String getURL(String cluster) {
-    String url = clusterMap.get(cluster);
-    return url;
-  }
-
   public Iterator<String> getClusters() {
-    Set<String> keys = clusterMap.keySet();
-    Iterator<String> i = keys.iterator();
+    Iterator<String> i = clusterMap.iterator();
     return i;
   }
 }

Modified: incubator/chukwa/trunk/src/web/hicc/jsp/cluster_selector.jsp
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/web/hicc/jsp/cluster_selector.jsp?rev=1050592&r1=1050591&r2=1050592&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/web/hicc/jsp/cluster_selector.jsp (original)
+++ incubator/chukwa/trunk/src/web/hicc/jsp/cluster_selector.jsp Sat Dec 18 07:34:59 2010
@@ -38,7 +38,6 @@ Cluster
            }
        }
        ClusterConfig cc = new ClusterConfig();
-       String jdbc = cc.getURL(cluster);
        Iterator i = cc.getClusters();
        while(i.hasNext()) { 
            String label = i.next().toString();

Modified: incubator/chukwa/trunk/src/web/hicc/jsp/event2_viewer.jsp
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/web/hicc/jsp/event2_viewer.jsp?rev=1050592&r1=1050591&r2=1050592&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/web/hicc/jsp/event2_viewer.jsp (original)
+++ incubator/chukwa/trunk/src/web/hicc/jsp/event2_viewer.jsp Sat Dec 18 07:34:59 2010
@@ -37,7 +37,6 @@ Filter: <input type="text" id="<%= XssFi
 <%
         String cluster = (String) session.getAttribute("cluster");
         ClusterConfig cc = new ClusterConfig();
-        String jdbc = cc.getURL(cluster);
         String boxId=XssFilter.filter(request.getParameter("boxId"));
         TimeHandler time = new TimeHandler(request);
         String startdate = time.getStartTimeText();



Mime
View raw message