chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r1140832 - in /incubator/chukwa/trunk: ./ conf/ ivy/ src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/ src/test/ src/test/org/apache/hadoop/chukwa/datacollection/writer/
Date Tue, 28 Jun 2011 20:47:34 GMT
Author: eyang
Date: Tue Jun 28 20:47:33 2011
New Revision: 1140832

URL: http://svn.apache.org/viewvc?rev=1140832&view=rev
Log:
CHUKWA-592.  Enable test case for HBaseWriter. (Eric Yang)

Modified:
    incubator/chukwa/trunk/conf/hbase.schema
    incubator/chukwa/trunk/ivy.xml
    incubator/chukwa/trunk/ivy/libraries.properties
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
    incubator/chukwa/trunk/src/test/hbase-site.xml
    incubator/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/writer/TestHBaseWriter.java

Modified: incubator/chukwa/trunk/conf/hbase.schema
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/conf/hbase.schema?rev=1140832&r1=1140831&r2=1140832&view=diff
==============================================================================
--- incubator/chukwa/trunk/conf/hbase.schema (original)
+++ incubator/chukwa/trunk/conf/hbase.schema Tue Jun 28 20:47:33 2011
@@ -27,4 +27,6 @@ create "ClusterSummary", {NAME=> "cpu", 
 {NAME => "network", VERSION => 65535},
 {NAME => "hdfs", VERSION => 65535},
 {NAME => "mapreduce", VERSION => 65535}
-
+create "chukwa", {NAME=>"chukwaAgent_chunkQueue", VERSIONS => 65535},
+{NAME => "chukwaAgent_metrics", VERSION => 65535},
+{NAME => "chukwaAgent_httpSender", VERSION => 65535}

Modified: incubator/chukwa/trunk/ivy.xml
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/ivy.xml?rev=1140832&r1=1140831&r2=1140832&view=diff
==============================================================================
--- incubator/chukwa/trunk/ivy.xml (original)
+++ incubator/chukwa/trunk/ivy.xml Tue Jun 28 20:47:33 2011
@@ -76,10 +76,10 @@
       name="guava"
       rev="${guava.version}"
       conf="common->master"/>
-    <!-- <dependency org="org.apache.hadoop.zookeeper"
+    <dependency org="org.apache.zookeeper"
       name="zookeeper"
       rev="${zookeeper.version}"
-      conf="common->master"/> -->
+      conf="common->master"/>
     <dependency org="com.sun.jersey"
       name="jersey-core"
       rev="${jersey.version}"/>

Modified: incubator/chukwa/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/ivy/libraries.properties?rev=1140832&r1=1140831&r2=1140832&view=diff
==============================================================================
--- incubator/chukwa/trunk/ivy/libraries.properties (original)
+++ incubator/chukwa/trunk/ivy/libraries.properties Tue Jun 28 20:47:33 2011
@@ -52,7 +52,7 @@ xerces.version=1.4.4
 #jaxb.version=2.1.12
 hbase.version=0.90.3
 thrift.version=0.2.0
-zookeeper.version=3.2.2
+zookeeper.version=3.3.3
 jersey.version=1.1.5.1
 guava.version=r05
 #avro.version=1.3.2

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java?rev=1140832&r1=1140831&r2=1140832&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
(original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
Tue Jun 28 20:47:33 2011
@@ -100,7 +100,7 @@ public class HBaseWriter extends Pipelin
     hconf = HBaseConfiguration.create();
   }
 
-  public HBaseWriter(ChukwaConfiguration conf, HBaseConfiguration hconf) {
+  public HBaseWriter(ChukwaConfiguration conf, Configuration hconf) {
     this(true);
     this.conf = conf;
     this.hconf = hconf;

Modified: incubator/chukwa/trunk/src/test/hbase-site.xml
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/test/hbase-site.xml?rev=1140832&r1=1140831&r2=1140832&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/test/hbase-site.xml (original)
+++ incubator/chukwa/trunk/src/test/hbase-site.xml Tue Jun 28 20:47:33 2011
@@ -32,15 +32,17 @@
   </property>
   <property>
     <name>hbase.client.pause</name>
-    <value>5000</value>
+    <value>1000</value>
     <description>General client pause value.  Used mostly as value to wait
     before running a retry of a failed get, region lookup, etc.</description>
   </property>
   <property>
-    <name>hbase.master.meta.thread.rescanfrequency</name>
-    <value>10000</value>
-    <description>How long the HMaster sleeps (in milliseconds) between scans of
-    the root and meta tables.
+    <name>hbase.client.retries.number</name>
+    <value>10</value>
+    <description>Maximum retries.  Used as maximum for all retryable
+    operations such as fetching of the root region from root region
+    server, getting a cell's value, starting a row update, etc.
+    Default: 10.
     </description>
   </property>
   <property>
@@ -51,6 +53,12 @@
     </description>
   </property>
   <property>
+    <name>hbase.master.event.waiting.time</name>
+    <value>50</value>
+    <description>Time to sleep between checks to see if a table event took place.
+    </description>
+  </property>
+  <property>
     <name>hbase.regionserver.handler.count</name>
     <value>5</value>
     <description>Count of RPC Server instances spun up on RegionServers
@@ -59,15 +67,6 @@
     </description>
   </property>
   <property>
-    <name>hbase.master.lease.period</name>
-    <value>6000</value>
-    <description>Length of time the master will wait before timing out a region
-    server lease. Since region servers report in every second (see above), this
-    value has been reduced so that the master will notice a dead region server
-    sooner. The default is 30 seconds.
-    </description>
-  </property>
-  <property>
     <name>hbase.master.info.port</name>
     <value>-1</value>
     <description>The port for the hbase master web UI
@@ -99,7 +98,7 @@
   </property>
   <property>
     <name>hbase.regionserver.optionalcacheflushinterval</name>
-    <value>10000</value>
+    <value>1000</value>
     <description>
     Amount of time to wait since the last time a region was flushed before
     invoking an optional cache flush. Default 60,000.
@@ -129,9 +128,22 @@
   </property>
   <property>
     <name>hbase.zookeeper.property.clientPort</name>
-    <value>21810</value>
+    <value>21818</value>
     <description>Property from ZooKeeper's config zoo.cfg.
     The port at which the clients will connect.
     </description>
   </property>
+  <property>
+    <name>hbase.defaults.for.version.skip</name>
+    <value>true</value>
+    <description>
+    Set to true to skip the 'hbase.defaults.for.version'.
+    Setting this to true can be useful in contexts other than
+    the other side of a maven generation; i.e. running in an
+    ide.  You'll want to set this boolean to true to avoid
+    seeing the RuntimException complaint: "hbase-default.xml file
+    seems to be for and old version of HBase (@@@VERSION@@@), this
+    version is X.X.X-SNAPSHOT"
+    </description>
+  </property>
 </configuration>

Modified: incubator/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/writer/TestHBaseWriter.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/writer/TestHBaseWriter.java?rev=1140832&r1=1140831&r2=1140832&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/writer/TestHBaseWriter.java
(original)
+++ incubator/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/writer/TestHBaseWriter.java
Tue Jun 28 20:47:33 2011
@@ -29,7 +29,7 @@ import org.apache.hadoop.chukwa.conf.Chu
 import org.apache.hadoop.chukwa.datacollection.writer.hbase.HBaseWriter;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
-//import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
@@ -42,72 +42,72 @@ import org.apache.log4j.Logger;
 
 public class TestHBaseWriter extends TestCase{
   static Logger log = Logger.getLogger(TestHBaseWriter.class);
-//  private HBaseTestingUtility util;
-//  private HBaseWriter hbw;
-//  private Configuration conf;
-//  private byte[] columnFamily = Bytes.toBytes("TestColumnFamily");
-//  private byte[] qualifier = Bytes.toBytes("Key");
-//  private byte[] expectedValue = Bytes.toBytes("Value");
-//
-//  private byte[] table = Bytes.toBytes("Test");
-//  private byte[] test = Bytes.toBytes("1234567890 Key Value");
-//  private ChukwaConfiguration cc;
-//  long timestamp = 1234567890;
-//  
-//  public TestHBaseWriter() {
-//    cc = new ChukwaConfiguration();
-//
-//    conf = HBaseConfiguration.create();
-//    conf.set("hbase.hregion.memstore.flush.size", String.valueOf(128*1024));
-//    try {
-//      util = new HBaseTestingUtility(conf);
-//      util.startMiniZKCluster();
-//      util.getConfiguration().setBoolean("dfs.support.append", true);
-//      util.startMiniCluster(2);
-//      HTableDescriptor desc = new HTableDescriptor();
-//      HColumnDescriptor family = new HColumnDescriptor(columnFamily);
-//      desc.setName(table);
-//      desc.addFamily(family);
-//      util.getHBaseAdmin().createTable(desc);
-//
-//    } catch (Exception e) {
-//      e.printStackTrace();
-//      Assert.fail(e.getMessage());
-//    }
-//  }
-//  
-//  public void setup() {
-//    
-//  }
-//  
-//  public void tearDown() {
-//    
-//  }
-//  
-//  public void testWriters() {
-//    ArrayList<Chunk> chunks = new ArrayList<Chunk>();
-//    chunks.add(new ChunkImpl("TextParser", "name", timestamp, test, null));      
-//    try {      
-//      cc.set("hbase.demux.package", "org.apache.chukwa.datacollection.writer.test.demux");
-//      cc.set("TextParser","org.apache.hadoop.chukwa.datacollection.writer.test.demux.TextParser");
-//      conf.set(HConstants.ZOOKEEPER_QUORUM, "127.0.0.1");
-//      hbw = new HBaseWriter(cc, conf);
-//      hbw.init(cc);
-//      if(hbw.add(chunks)!=ChukwaWriter.COMMIT_OK) {
-//        Assert.fail("Commit status is not OK.");
-//      }
-//      HTable testTable = new HTable(table);
-//      ResultScanner scanner = testTable.getScanner(columnFamily, qualifier);
-//      for(Result res : scanner) {
-//        Assert.assertEquals(new String(expectedValue), new String(res.getValue(columnFamily,
qualifier)));
-//      }
-//      // Cleanup and return
-//      scanner.close();
-//      // Compare data in Hbase with generated chunks
-//      util.shutdownMiniCluster();
-//    } catch (Exception e) {
-//      e.printStackTrace();
-//      Assert.fail(e.getMessage());
-//    }
-//  }
+  private HBaseTestingUtility util;
+  private HBaseWriter hbw;
+  private Configuration conf;
+  private byte[] columnFamily = Bytes.toBytes("TestColumnFamily");
+  private byte[] qualifier = Bytes.toBytes("Key");
+  private byte[] expectedValue = Bytes.toBytes("Value");
+
+  private byte[] table = Bytes.toBytes("Test");
+  private byte[] test = Bytes.toBytes("1234567890 Key Value");
+  private ChukwaConfiguration cc;
+  long timestamp = 1234567890;
+  
+  public TestHBaseWriter() {
+    cc = new ChukwaConfiguration();
+
+    conf = HBaseConfiguration.create();
+    conf.set("hbase.hregion.memstore.flush.size", String.valueOf(128*1024));
+    try {
+      util = new HBaseTestingUtility(conf);
+      util.startMiniZKCluster();
+      util.getConfiguration().setBoolean("dfs.support.append", true);
+      util.startMiniCluster(2);
+      HTableDescriptor desc = new HTableDescriptor();
+      HColumnDescriptor family = new HColumnDescriptor(columnFamily);
+      desc.setName(table);
+      desc.addFamily(family);
+      util.getHBaseAdmin().createTable(desc);
+
+    } catch (Exception e) {
+      e.printStackTrace();
+      Assert.fail(e.getMessage());
+    }
+  }
+  
+  public void setup() {
+    
+  }
+  
+  public void tearDown() {
+    
+  }
+  
+  public void testWriters() {
+    ArrayList<Chunk> chunks = new ArrayList<Chunk>();
+    chunks.add(new ChunkImpl("TextParser", "name", timestamp, test, null));      
+    try {      
+      cc.set("hbase.demux.package", "org.apache.chukwa.datacollection.writer.test.demux");
+      cc.set("TextParser","org.apache.hadoop.chukwa.datacollection.writer.test.demux.TextParser");
+      conf.set(HConstants.ZOOKEEPER_QUORUM, "127.0.0.1");
+      hbw = new HBaseWriter(cc, conf);
+      hbw.init(cc);
+      if(hbw.add(chunks)!=ChukwaWriter.COMMIT_OK) {
+        Assert.fail("Commit status is not OK.");
+      }
+      HTable testTable = new HTable(table);
+      ResultScanner scanner = testTable.getScanner(columnFamily, qualifier);
+      for(Result res : scanner) {
+        Assert.assertEquals(new String(expectedValue), new String(res.getValue(columnFamily,
qualifier)));
+      }
+      // Cleanup and return
+      scanner.close();
+      // Compare data in Hbase with generated chunks
+      util.shutdownMiniCluster();
+    } catch (Exception e) {
+      e.printStackTrace();
+      Assert.fail(e.getMessage());
+    }
+  }
 }



Mime
View raw message