hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject svn commit: r720595 - in /hadoop/hbase/branches/0.19_on_hadoop_0.18: ./ conf/ src/java/ src/java/org/apache/hadoop/hbase/client/tableindexed/ src/java/org/apache/hadoop/hbase/regionserver/ src/java/org/apache/hadoop/hbase/regionserver/metrics/
Date Tue, 25 Nov 2008 21:05:01 GMT
Author: apurtell
Date: Tue Nov 25 13:05:00 2008
New Revision: 720595

URL: http://svn.apache.org/viewvc?rev=720595&view=rev
Log:
merge up to trunk (revision 720575)

Modified:
    hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt
    hadoop/hbase/branches/0.19_on_hadoop_0.18/conf/hadoop-metrics.properties
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexSpecification.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTable.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/overview.html

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt?rev=720595&r1=720594&r2=720595&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt Tue Nov 25 13:05:00 2008
@@ -77,6 +77,12 @@
    HBASE-1003  If cell exceeds TTL but not VERSIONs, will not be removed during
                major compaction
    HBASE-1005  Regex and string comparison operators for ColumnValueFilter
+   HBASE-910   Scanner misses columns / rows when the scanner is obtained
+               durring a memcache flush
+   HBASE-1009  Master stuck in loop wanting to assign but regions are closing
+   HBASE-1016  Fix example in javadoc overvie
+   HBASE-1021  hbase metrics FileContext not working
+   HBASE-1023  Check global flusher
       
   IMPROVEMENTS
    HBASE-901   Add a limit to key length, check key and value length on client side
@@ -129,7 +135,12 @@
                the deleted cell
    HBASE-675   Report correct server hosting a table split for assignment to
                for MR Jobs
-        
+   HBASE-927   We don't recover if HRS hosting -ROOT-/.META. goes down
+   HBASE-1013  Add debugging around commit log cleanup
+   HBASE-972   Update hbase trunk to use released hadoop 0.19.0
+   HBASE-1022  Add storefile index size to hbase metrics
+   HBASE-1026  Tests in mapred are failing
+ 
   NEW FEATURES
    HBASE-875   Use MurmurHash instead of JenkinsHash [in bloomfilters]
                (Andrzej Bialecki via Stack)

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/conf/hadoop-metrics.properties
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/conf/hadoop-metrics.properties?rev=720595&r1=720594&r2=720595&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/conf/hadoop-metrics.properties (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/conf/hadoop-metrics.properties Tue Nov 25 13:05:00
2008
@@ -8,7 +8,7 @@
 hbase.class=org.apache.hadoop.metrics.spi.NullContext
 
 # Configuration of the "hbase" context for file
-# hbase.class=org.apache.hadoop.metrics.file.FileContext
+# hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
 # hbase.period=10
 # hbase.fileName=/tmp/metrics_hbase.log
 
@@ -21,7 +21,7 @@
 jvm.class=org.apache.hadoop.metrics.spi.NullContext
 
 # Configuration of the "jvm" context for file
-# jvm.class=org.apache.hadoop.metrics.file.FileContext
+# jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
 # jvm.period=10
 # jvm.fileName=/tmp/metrics_jvm.log
 
@@ -31,10 +31,10 @@
 # jvm.servers=GMETADHOST_IP:8649
 
 # Configuration of the "rpc" context for null
-hbase.class=org.apache.hadoop.metrics.spi.NullContext
+rpc.class=org.apache.hadoop.metrics.spi.NullContext
 
 # Configuration of the "rpc" context for file
-# rpc.class=org.apache.hadoop.metrics.file.FileContext
+# rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
 # rpc.period=10
 # rpc.fileName=/tmp/metrics_rpc.log
 

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexSpecification.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexSpecification.java?rev=720595&r1=720594&r2=720595&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexSpecification.java
(original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexSpecification.java
Tue Nov 25 13:05:00 2008
@@ -59,7 +59,6 @@
    * @param indexedColumns
    * @param additionalColumns
    * @param keyGenerator
-   * @param keyComparator
    */
   public IndexSpecification(String indexId, byte[][] indexedColumns,
       byte[][] additionalColumns, IndexKeyGenerator keyGenerator) {

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTable.java?rev=720595&r1=720594&r2=720595&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTable.java
(original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTable.java
Tue Nov 25 13:05:00 2008
@@ -52,7 +52,6 @@
 
   private Map<String, HTable> indexIdToTable = new HashMap<String, HTable>();
 
-  /** {@inheritDoc} */
   public IndexedTable(final HBaseConfiguration conf, final byte[] tableName)
       throws IOException {
     super(conf, tableName);

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=720595&r1=720594&r2=720595&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
(original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Tue Nov 25 13:05:00 2008
@@ -297,6 +297,7 @@
       // Now ask master what it wants us to do and tell it what we have done
       for (int tries = 0; !stopRequested.get() && isHealthy();) {
         // Try to get the root region location from the master.
+        if (!haveRootRegion.get()) {
           HServerAddress rootServer = hbaseMaster.getRootRegionLocation();
           if (rootServer != null) {
             // By setting the root region location, we bypass the wait imposed on
@@ -305,6 +306,7 @@
                 new HRegionLocation(HRegionInfo.ROOT_REGIONINFO, rootServer));
             haveRootRegion.set(true);
           }
+        }
           long now = System.currentTimeMillis();
         if (lastMsg != 0 && (now - lastMsg) >= serverLeaseTimeout) {
           // It has been way too long since we last reported to the master.
@@ -650,21 +652,33 @@
     // Is this too expensive every three seconds getting a lock on onlineRegions
     // and then per store carried?  Can I make metrics be sloppier and avoid
     // the synchronizations?
+    int stores = 0;
     int storefiles = 0;
     long memcacheSize = 0;
+    long storefileIndexSize = 0;
     synchronized (this.onlineRegions) {
       for (Map.Entry<Integer, HRegion> e: this.onlineRegions.entrySet()) {
         HRegion r = e.getValue();
         memcacheSize += r.memcacheSize.get();
         synchronized(r.stores) {
+          stores += r.stores.size();
           for(Map.Entry<Integer, HStore> ee: r.stores.entrySet()) {
-            storefiles += ee.getValue().getStorefilesCount();
+            HStore store = ee.getValue(); 
+            storefiles += store.getStorefilesCount();
+            try {
+              storefileIndexSize += store.getStorefilesIndexSize();
+            } catch (IOException ex) {
+              LOG.warn("error getting store file index size for " + store +
+                ": " + StringUtils.stringifyException(ex));  
+            }
           }
         }
       }
     }
+    this.metrics.stores.set(stores);
     this.metrics.storefiles.set(storefiles);
     this.metrics.memcacheSizeMB.set((int)(memcacheSize/(1024*1024)));
+    this.metrics.storefileIndexSizeMB.set((int)(storefileIndexSize/(1024*1024)));
   }
 
   /**

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=720595&r1=720594&r2=720595&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
(original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
Tue Nov 25 13:05:00 2008
@@ -2137,7 +2137,19 @@
   int getStorefilesCount() {
     return this.storefiles.size();
   }
-  
+
+  /**
+   * @return The size of the store file indexes, in bytes.
+   * @throws IOException if there was a problem getting file sizes from the
+   * filesystem
+   */
+  long getStorefilesIndexSize() throws IOException {
+    long size = 0;
+    for (HStoreFile s: storefiles.values())
+      size += s.indexLength();
+    return size;
+  }
+
   /*
    * Datastructure that holds size and key.
    */

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java?rev=720595&r1=720594&r2=720595&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
(original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
Tue Nov 25 13:05:00 2008
@@ -52,7 +52,7 @@
  * <p>An HStoreFile usually tracks 4 things: its parent dir, the region
  * identifier, the column family, and the file identifier.  If you know those
  * four things, you know how to obtain the right HStoreFile.  HStoreFiles may
- * also refernce store files in another region serving either from
+ * also reference store files in another region serving either from
  * the top-half of the remote file or from the bottom-half.  Such references
  * are made fast splitting regions.
  * 
@@ -101,6 +101,7 @@
   /* If true, this file was product of a major compaction.
    */
   private boolean majorCompaction = false;
+  private long indexLength;
 
   /**
    * Constructor that fully initializes the object
@@ -381,7 +382,7 @@
       out.close();
     }
   }
-  
+
   /**
    * Delete store map files.
    * @throws IOException 
@@ -477,6 +478,18 @@
     return (isReference())? l / 2: l;
   }
 
+  /**
+   * @return Length of the store map file index.
+   * @throws IOException
+   */
+  public synchronized long indexLength() throws IOException {
+    if (indexLength == 0) {
+      Path p = new Path(getMapFilePath(reference), MapFile.INDEX_FILE_NAME);
+      indexLength = p.getFileSystem(conf).getFileStatus(p).getLen();
+    }
+    return indexLength;
+  }
+
   @Override
   public String toString() {
     return encodedRegionName + "/" + Bytes.toString(colFamily) + "/" + fileId +

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java?rev=720595&r1=720594&r2=720595&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java
(original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java
Tue Nov 25 13:05:00 2008
@@ -220,17 +220,23 @@
    */
   private synchronized void flushSomeRegions() {
     // keep flushing until we hit the low water mark
+    long globalMemcacheSize = -1;
     for (SortedMap<Long, HRegion> m =
         this.server.getCopyOfOnlineRegionsSortedBySize();
-      server.getGlobalMemcacheSize() >= globalMemcacheLimitLowMark;) {
+      (globalMemcacheSize = server.getGlobalMemcacheSize()) >=
+        this.globalMemcacheLimitLowMark;) {
       // flush the region with the biggest memcache
       if (m.size() <= 0) {
         LOG.info("No online regions to flush though we've been asked flush " +
-            "some; globalMemcacheSize=" + this.server.getGlobalMemcacheSize() +
+            "some; globalMemcacheSize=" + globalMemcacheSize +
             ", globalMemcacheLimitLowMark=" + this.globalMemcacheLimitLowMark);
         break;
       }
       HRegion biggestMemcacheRegion = m.remove(m.firstKey());
+      LOG.info("Forced flushing of " +  biggestMemcacheRegion.toString() +
+        " because global memcache limit of " + this.globalMemcacheLimit +
+        " exceeded; currenly " + globalMemcacheSize + " and flushing till " +
+        this.globalMemcacheLimitLowMark);
       if (!flushRegion(biggestMemcacheRegion, true)) {
         // Something bad happened - give up.
         break;

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=720595&r1=720594&r2=720595&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
(original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
Tue Nov 25 13:05:00 2008
@@ -48,12 +48,23 @@
    * Count of requests to the regionservers since last call to metrics update
    */
   private final MetricsIntValue requests = new MetricsIntValue("requests");
-  
+
+  /**
+   * Count of stores open on the regionserver.
+   */
+  public final MetricsIntValue stores = new MetricsIntValue("stores");
+
   /**
    * Count of storefiles open on the regionserver.
    */
   public final MetricsIntValue storefiles = new MetricsIntValue("storefiles");
-  
+
+  /**
+   * Sum of all the storefile index sizes in this regionserver in MB
+   */
+  public final MetricsIntValue storefileIndexSizeMB =
+    new MetricsIntValue("storefileIndexSizeMB");
+
   /**
    * Sum of all the memcache sizes in this regionserver in MB
    */
@@ -80,7 +91,9 @@
    */
   public void doUpdates(@SuppressWarnings("unused") MetricsContext unused) {
     synchronized (this) {
+      this.stores.pushMetric(this.metricsRecord);
       this.storefiles.pushMetric(this.metricsRecord);
+      this.storefileIndexSizeMB.pushMetric(this.metricsRecord);
       this.memcacheSizeMB.pushMetric(this.metricsRecord);
       this.regions.pushMetric(this.metricsRecord);
       synchronized(this.requests) {
@@ -124,8 +137,13 @@
     sb.append(this.requests.get()/seconds);
     sb.append(", regions=");
     sb.append(this.regions.get());
+    sb.append(", stores=");
+    sb.append(this.stores.get());
     sb.append(", storefiles=");
     sb.append(this.storefiles.get());
+    sb.append(", storefileIndexSize=");
+    sb.append(this.storefileIndexSizeMB.get());
+    sb.append("MB");
     sb.append(", memcacheSize=");
     sb.append(this.memcacheSizeMB.get());
     sb.append("MB");

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/overview.html
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/overview.html?rev=720595&r1=720594&r2=720595&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/overview.html (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/overview.html Tue Nov 25 13:05:00 2008
@@ -195,6 +195,7 @@
 import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.io.Cell;
 import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.hbase.util.Bytes;
 
 public class MyClient {
 
@@ -209,17 +210,20 @@
 
     // To do any sort of update on a row, you use an instance of the BatchUpdate
     // class. A BatchUpdate takes a row and optionally a timestamp which your
-    // updates will affect. 
+    // updates will affect.  If no timestamp, the server applies current time
+    // to the edits.
     BatchUpdate batchUpdate = new BatchUpdate("myRow");
 
-    // The BatchUpdate#put method takes a Text that describes what cell you want
-    // to put a value into, and a byte array that is the value you want to 
-    // store. Note that if you want to store strings, you have to getBytes() 
-    // from the string for HBase to understand how to store it. (The same goes
-    // for primitives like ints and longs and user-defined classes - you must 
-    // find a way to reduce it to bytes.)
+    // The BatchUpdate#put method takes a byte [] (or String) that designates
+    // what cell you want to put a value into, and a byte array that is the
+    // value you want to store. Note that if you want to store Strings, you
+    // have to getBytes() from the String for HBase to store it since HBase is
+    // all about byte arrays. The same goes for primitives like ints and longs
+    // and user-defined classes - you must find a way to reduce it to bytes.
+    // The Bytes class from the hbase util package has utility for going from
+    // String to utf-8 bytes and back again and help for other base types.
     batchUpdate.put("myColumnFamily:columnQualifier1", 
-      "columnQualifier1 value!".getBytes());
+      Bytes.toBytes("columnQualifier1 value!"));
 
     // Deletes are batch operations in HBase as well. 
     batchUpdate.delete("myColumnFamily:cellIWantDeleted");
@@ -235,7 +239,9 @@
     // value contained is a string and want an actual string, then you must 
     // convert it yourself.
     Cell cell = table.get("myRow", "myColumnFamily:columnQualifier1");
-    String valueStr = new String(cell.getValue());
+    // This could throw a NullPointerException if there was no value at the cell
+    // location.
+    String valueStr = Bytes.toString(cell.getValue());
     
     // Sometimes, you won't know the row you're looking for. In this case, you
     // use a Scanner. This will give you cursor-like interface to the contents
@@ -245,30 +251,31 @@
       table.getScanner(new String[]{"myColumnFamily:columnQualifier1"});
     
     
-    // Scanners in HBase 0.2 return RowResult instances. A RowResult is like the
-    // row key and the columns all wrapped up in a single interface. 
+    // Scanners return RowResult instances. A RowResult is like the
+    // row key and the columns all wrapped up in a single Object. 
     // RowResult#getRow gives you the row key. RowResult also implements 
     // Map, so you can get to your column results easily. 
     
     // Now, for the actual iteration. One way is to use a while loop like so:
     RowResult rowResult = scanner.next();
     
-    while(rowResult != null) {
+    while (rowResult != null) {
       // print out the row we found and the columns we were looking for
-      System.out.println("Found row: " + new String(rowResult.getRow()) + " with value: "
+
-       rowResult.get("myColumnFamily:columnQualifier1".getBytes()));
-      
+      System.out.println("Found row: " + Bytes.toString(rowResult.getRow()) +
+        " with value: " + rowResult.get(Bytes.toBytes("myColumnFamily:columnQualifier1")));
       rowResult = scanner.next();
     }
     
     // The other approach is to use a foreach loop. Scanners are iterable!
     for (RowResult result : scanner) {
       // print out the row we found and the columns we were looking for
-      System.out.println("Found row: " + new String(result.getRow()) + " with value: " +
-       result.get("myColumnFamily:columnQualifier1".getBytes()));
+      System.out.println("Found row: " + Bytes.toString(rowResult.getRow()) +
+        " with value: " + rowResult.get(Bytes.toBytes("myColumnFamily:columnQualifier1")));
     }
     
     // Make sure you close your scanners when you are done!
+    // Its probably best to put the iteration into a try/finally with the below
+    // inside the finally clause.
     scanner.close();
   }
 }



Mime
View raw message