hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mbau...@apache.org
Subject svn commit: r1366370 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/client/ main/java/org/apache/hadoop/hbase/ipc/ test/java/org/apache/hadoop/hbase/ipc/
Date Fri, 27 Jul 2012 12:50:19 GMT
Author: mbautin
Date: Fri Jul 27 12:50:19 2012
New Revision: 1366370

URL: http://svn.apache.org/viewvc?rev=1366370&view=rev
Log:
[HBASE-6215] per-request profiling merging

Author: aurickq

Summary: ability to merge profilingdata objects and change client-side code to aggregate profiling
data instead of clearing after every RPC call.

Test Plan: new Unit test

Reviewers: kranganathan, liujiakai

Reviewed By: kranganathan

CC: hbase-eng@

Differential Revision: https://phabricator.fb.com/D531435

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/ProfilingData.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/ipc/TestPerRequestProfiling.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=1366370&r1=1366369&r2=1366370&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java Fri Jul
27 12:50:19 2012
@@ -551,9 +551,13 @@ public class HTable implements HTableInt
     return connection.processBatchOfGets(gets, tableName, this.options);
   }
 
-  public ProfilingData getProfilingData () {
+  public ProfilingData getProfilingData() {
     return this.options.profilingResult;
   }
+  
+  public void clearProfilingData() {
+    this.options.profilingResult = null;
+  }
 
   /**
    * {@inheritDoc}

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java?rev=1366370&r1=1366369&r2=1366370&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java Fri
Jul 27 12:50:19 2012
@@ -595,12 +595,17 @@ public class HBaseClient {
         } else {
           Writable value = ReflectionUtils.newInstance(valueClass, conf);
           value.readFields(localIn);                 // read value
-          call.options.profilingResult = null;  // clear out previous results
           if (call.getVersion() >= HBaseServer.VERSION_RPCOPTIONS) {
-            boolean hasProfiling = localIn.readBoolean ();
+            boolean hasProfiling = localIn.readBoolean();
             if (hasProfiling) {
-              call.options.profilingResult = new ProfilingData();
-              call.options.profilingResult.readFields(localIn);
+              if (call.options.profilingResult == null) {
+                call.options.profilingResult = new ProfilingData();
+                call.options.profilingResult.readFields(localIn);
+              } else {
+                ProfilingData pData = new ProfilingData();
+                pData.readFields(localIn);
+                call.options.profilingResult.merge(pData);
+              }
               Long serverTimeObj = call.options.profilingResult.getLong(
                   ProfilingData.TOTAL_SERVER_TIME_MS);
               if (serverTimeObj != null) {

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/ProfilingData.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/ProfilingData.java?rev=1366370&r1=1366369&r2=1366370&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/ProfilingData.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/ProfilingData.java Fri
Jul 27 12:50:19 2012
@@ -59,6 +59,11 @@ public class ProfilingData implements Wr
    *  name of the rpc method called
    */
   public static final String RPC_METHOD_NAME = "rpc_method_name";
+  
+  /**
+   *  separator used when concatenating strings to be merged
+   */
+  public static final String STRING_MERGE_SEPARATOR = ",";
 
 	private Map<String, String> mapString = new HashMap<String, String>();
 	private Map<String, MutableLong> mapLong = new HashMap<String, MutableLong>();
@@ -174,31 +179,87 @@ public class ProfilingData implements Wr
   public void decFloat(String key, float amt) {
     this.incFloat(key, -amt);
   }
+  
+  public void merge(ProfilingData pData) {
+    // TODO: this just adds/concatenates data, maybe will need
+    // to handle other merge types, eg max, min, average
+    for (Map.Entry<String, String> entry : pData.mapString.entrySet()) {
+      String key = entry.getKey();
+      String val = entry.getValue();
+      String base = this.mapString.get(key);
+      if (base == null) {
+        this.mapString.put(key, val);
+      } else {
+        this.mapString.put(key, base + 
+            ProfilingData.STRING_MERGE_SEPARATOR + val);
+      }
+    }
+    for (Map.Entry<String, Boolean> entry : pData.mapBoolean.entrySet()) {
+      String key = entry.getKey();
+      boolean val = entry.getValue();
+      Boolean base = this.mapBoolean.get(key);
+      if (base == null) {
+        this.mapBoolean.put(key, val);
+      } else {
+        this.mapBoolean.put(key, base || val);
+      }
+    }
+    for (Map.Entry<String, MutableInt> entry : pData.mapInt.entrySet()) {
+      String key = entry.getKey();
+      MutableInt val = entry.getValue();
+      MutableInt base = this.mapInt.get(key);
+      if (base == null) {
+        this.mapInt.put(key, new MutableInt(val.intValue()));
+      } else {
+        base.add(val);
+      }
+    }
+    for (Map.Entry<String, MutableLong> entry : pData.mapLong.entrySet()) {
+      String key = entry.getKey();
+      MutableLong val = entry.getValue();
+      MutableLong base = this.mapLong.get(key);
+      if (base == null) {
+        this.mapLong.put(key, new MutableLong(val.longValue()));
+      } else {
+        base.add(val);
+      }
+    }
+    for (Map.Entry<String, MutableFloat> entry : pData.mapFloat.entrySet()) {
+      String key = entry.getKey();
+      MutableFloat val = entry.getValue();
+      MutableFloat base = this.mapFloat.get(key);
+      if (base == null) {
+        this.mapFloat.put(key, new MutableFloat(val.floatValue()));
+      } else {
+        base.add(val);
+      }
+    }
+  }
 	
 	@Override
 	public void write(DataOutput out) throws IOException {
 	  out.writeInt(mapString.size());
-	  for (Map.Entry<String,String> entry : mapString.entrySet ()) {
+	  for (Map.Entry<String,String> entry : mapString.entrySet()) {
       out.writeUTF(entry.getKey());
       out.writeUTF(entry.getValue());
     }
 	  out.writeInt(mapBoolean.size());
-    for (Map.Entry<String,Boolean> entry : mapBoolean.entrySet ()) {
+    for (Map.Entry<String,Boolean> entry : mapBoolean.entrySet()) {
       out.writeUTF(entry.getKey());
       out.writeBoolean(entry.getValue());
     }
     out.writeInt(mapInt.size());
-    for (Map.Entry<String,MutableInt> entry : mapInt.entrySet ()) {
+    for (Map.Entry<String,MutableInt> entry : mapInt.entrySet()) {
       out.writeUTF(entry.getKey());
       out.writeInt(entry.getValue().intValue());
     }
     out.writeInt(mapLong.size());
-    for (Map.Entry<String,MutableLong> entry : mapLong.entrySet ()) {
+    for (Map.Entry<String,MutableLong> entry : mapLong.entrySet()) {
       out.writeUTF(entry.getKey());
       out.writeLong(entry.getValue().longValue());
     }
     out.writeInt(mapFloat.size());
-    for (Map.Entry<String,MutableFloat> entry : mapFloat.entrySet ()) {
+    for (Map.Entry<String,MutableFloat> entry : mapFloat.entrySet()) {
       out.writeUTF(entry.getKey());
       out.writeFloat(entry.getValue().floatValue());
     }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/ipc/TestPerRequestProfiling.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/ipc/TestPerRequestProfiling.java?rev=1366370&r1=1366369&r2=1366370&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/ipc/TestPerRequestProfiling.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/ipc/TestPerRequestProfiling.java
Fri Jul 27 12:50:19 2012
@@ -86,6 +86,7 @@ public class TestPerRequestProfiling {
 
     LOG.debug("Testing with profiling off");
     // put some values with profiling off
+    table.clearProfilingData();
     byte [][] ROWS = { Bytes.toBytes("a"), Bytes.toBytes("b") };
     for (int i = 0; i < ROWS.length; i++) {
       Put put = new Put(ROWS[i]);
@@ -123,6 +124,7 @@ public class TestPerRequestProfiling {
       table.put(put);
       // autoflush is on by default, or else move this check after flush
       assertTrue (table.getProfilingData () != null);
+      table.clearProfilingData();
     }
     LOG.debug("Wrote some puts to table " + new String(TABLE));
 
@@ -146,7 +148,7 @@ public class TestPerRequestProfiling {
     // turn profiling back off and repeat test to make sure
     // profiling data gets cleared
     table.setProfiling(false);
-    
+    table.clearProfilingData();
     LOG.debug("Testing with profiling off");
     for (int i = 0; i < ROWS.length; i++) {
       Put put = new Put(ROWS[i]);
@@ -174,4 +176,61 @@ public class TestPerRequestProfiling {
     }
     LOG.debug("Read and verified from table " + new String(TABLE));
   }
+  
+  @Test
+  public void testProfilingData() throws Exception {
+    LOG.debug("Testing ProfilingData object");
+    
+    ProfilingData pData1 = new ProfilingData();
+    pData1.addString("testStringKey", "testStringVal");
+    pData1.addBoolean("testBooleanKey", false);
+    pData1.addInt("testIntKey", 1);
+    pData1.addLong("testLongKey", 2);
+    pData1.addFloat("testFloatKey", 3);
+    assertTrue(pData1.getString("testStringKey").equals("testStringVal"));
+    assertTrue(pData1.getBoolean("testBooleanKey") == false);
+    assertTrue(pData1.getInt("testIntKey") == 1);
+    assertTrue(pData1.getLong("testLongKey") == 2);
+    assertTrue(pData1.getFloat("testFloatKey") == 3);
+    pData1.incInt("testIntKey", 3);
+    pData1.incInt("testIntKey");
+    pData1.decInt("testIntKey", 5);
+    pData1.decInt("testIntKey");
+    pData1.incLong("testLongKey", 3);
+    pData1.incLong("testLongKey");
+    pData1.decLong("testLongKey", 5);
+    pData1.decLong("testLongKey");
+    pData1.incFloat("testFloatKey", 3);
+    pData1.decFloat("testFloatKey", 5);
+    assertTrue(pData1.getInt("testIntKey") == -1);
+    assertTrue(pData1.getLong("testLongKey") == 0);
+    assertTrue(pData1.getFloat("testFloatKey") == 1);
+    
+    ProfilingData pData2 = new ProfilingData();
+    pData2.addString("testStringKey", "2");
+    pData2.addString("testStringKey2", "testStringVal");
+    pData2.addBoolean("testBooleanKey", true);
+    pData2.addBoolean("testBooleanKey2", false);
+    pData2.addInt("testIntKey", 4);
+    pData2.addInt("testIntKey2", 1);
+    pData2.addLong("testLongKey", 5);
+    pData2.addLong("testLongKey2", 2);
+    pData2.addFloat("testFloatKey", 6);
+    pData2.addFloat("testFloatKey2", 3);
+    
+    pData1.merge(pData2);
+    assertTrue(pData1.getString("testStringKey").equals("testStringVal" + 
+          ProfilingData.STRING_MERGE_SEPARATOR + "2"));
+    assertTrue(pData1.getBoolean("testBooleanKey") == true);
+    assertTrue(pData1.getInt("testIntKey") == 3);
+    assertTrue(pData1.getLong("testLongKey") == 5);
+    assertTrue(pData1.getFloat("testFloatKey") == 7);
+    assertTrue(pData1.getString("testStringKey2").equals("testStringVal"));
+    assertTrue(pData1.getBoolean("testBooleanKey2") == false);
+    assertTrue(pData1.getInt("testIntKey2") == 1);
+    assertTrue(pData1.getLong("testLongKey2") == 2);
+    assertTrue(pData1.getFloat("testFloatKey2") == 3);
+    
+    LOG.debug("Finished testing ProfilingData object");
+  }
 }



Mime
View raw message