chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r1042449 - in /incubator/chukwa/trunk: CHANGES.txt conf/chukwa-collector-conf.xml.template src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
Date Sun, 05 Dec 2010 21:09:22 GMT
Author: eyang
Date: Sun Dec  5 21:09:22 2010
New Revision: 1042449

URL: http://svn.apache.org/viewvc?rev=1042449&view=rev
Log:
CHUKWA-563. Added configuration to skip HBase schema validation for HBaseWriter. (Eric Yang)


Modified:
    incubator/chukwa/trunk/CHANGES.txt
    incubator/chukwa/trunk/conf/chukwa-collector-conf.xml.template
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java

Modified: incubator/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/CHANGES.txt?rev=1042449&r1=1042448&r2=1042449&view=diff
==============================================================================
--- incubator/chukwa/trunk/CHANGES.txt (original)
+++ incubator/chukwa/trunk/CHANGES.txt Sun Dec  5 21:09:22 2010
@@ -4,6 +4,8 @@ Trunk (unreleased changes)
 
   NEW FEATURES
 
+    CHUKWA-563. Added configuration to skip HBase schema validation for HBaseWriter. (Eric
Yang)
+
     CHUKWA-558. Update default Chukwa configuration files to use SocketAdaptor for Hadoop
metrics streaming. (Eric Yang)
 
     CHUKWA-527  Configuration spellcheck. (Ari Rabkin)

Modified: incubator/chukwa/trunk/conf/chukwa-collector-conf.xml.template
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/conf/chukwa-collector-conf.xml.template?rev=1042449&r1=1042448&r2=1042449&view=diff
==============================================================================
--- incubator/chukwa/trunk/conf/chukwa-collector-conf.xml.template (original)
+++ incubator/chukwa/trunk/conf/chukwa-collector-conf.xml.template Sun Dec  5 21:09:22 2010
@@ -59,7 +59,7 @@
 
   <property>
     <name>hbase.zookeeper.quorum</name>
-    <value>host1.mydomain.com,host2.mydomain.com,host3.mydomain.com</value>
+    <value>localhost</value>
     <description>Comma separated list of servers in the ZooKeeper Quorum.
     For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
     By default this is set to localhost for local and pseudo-distributed modes
@@ -69,6 +69,22 @@
     </description>
   </property>
 
+  <property>
+    <name>hbase.writer.verify.schema</name>
+    <value>false</value>
+    <description>Verify HBase Table schema with demux parser schema, log
+    warning if there are mismatch between hbase schema and demux parsers.
+    </description>
+  </property>
+
+  <property>
+    <name>hbase.writer.halt.on.schema.mismatch</name>
+    <value>false</value>
+    <description>If this option is set to true, and HBase table schema 
+    is mismatched with demux parser, collector will shut down itself.
+    </description>
+  </property>
+
 -->
 
   <property>

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java?rev=1042449&r1=1042448&r2=1042449&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
(original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
Sun Dec  5 21:09:22 2010
@@ -113,7 +113,9 @@ public class HBaseWriter extends Pipelin
     statTimer.schedule(new StatReportingTask(), 1000, 10 * 1000);
     output = new OutputCollector();
     reporter = new Reporter();
-    verifyHbaseSchema();
+    if(conf.getBoolean("hbase.writer.verify.schema", false)) {
+      verifyHbaseSchema();      
+    }
     pool = new HTablePool(hconf, 60);
   }
 
@@ -141,7 +143,7 @@ public class HBaseWriter extends Pipelin
   
   private void verifyHbaseSchema() {
     log.debug("Verify Demux parser with HBase schema");
-    boolean schemaVerified = false;
+    boolean schemaVerified = true;
     try {
       HBaseAdmin admin = new HBaseAdmin(hconf);
       List<Class> demuxParsers = ClassUtils.getClassesForPackage(conf.get("hbase.demux.package"));
@@ -150,23 +152,25 @@ public class HBaseWriter extends Pipelin
           Tables list = x.getAnnotation(Tables.class);
           for(Table table : list.annotations()) {
             if(!verifyHbaseTable(admin, table)) {
-              throw new Exception("Validation failed - table: "+table.name()+" column family:
"+table.columnFamily()+" does not exist.");              
+              schemaVerified = false;
+              log.warn("Validation failed - table: "+table.name()+" column family: "+table.columnFamily()+"
does not exist.");              
             }
           }
         } else if(x.isAnnotationPresent(Table.class)) {
           Table table = x.getAnnotation(Table.class);
           if(!verifyHbaseTable(admin, table)) {
-            throw new Exception("Validation failed - table: "+table.name()+" column family:
"+table.columnFamily()+" does not exist.");
+            schemaVerified = false;
+            log.warn("Validation failed - table: "+table.name()+" column family: "+table.columnFamily()+"
does not exist.");
           }
         }
       }
-      schemaVerified = true;
     } catch (Exception e) {
+      schemaVerified = false;
       log.error(ExceptionUtil.getStackTrace(e));
     }
     if(!schemaVerified) {
       log.error("Hbase schema mismatch with demux parser.");
-      if(conf.getBoolean("halt.on.schema.mismatch", true)) {
+      if(conf.getBoolean("hbase.writer.halt.on.schema.mismatch", true)) {
         log.error("Exiting...");
         DaemonWatcher.bailout(-1);
       }



Mime
View raw message