hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject svn commit: r775816 - in /hadoop/hbase/trunk_on_hadoop-0.18.3: ./ src/java/org/apache/hadoop/hbase/client/ src/java/org/apache/hadoop/hbase/master/ src/java/org/apache/hadoop/hbase/regionserver/ src/test/org/apache/hadoop/hbase/client/
Date Mon, 18 May 2009 05:33:18 GMT
Author: apurtell
Date: Mon May 18 05:33:17 2009
New Revision: 775816

URL: http://svn.apache.org/viewvc?rev=775816&view=rev
Log:
HBASE-1431, HBASE-1401

Modified:
    hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestHTable.java

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt?rev=775816&r1=775815&r2=775816&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt Mon May 18 05:33:17 2009
@@ -131,6 +131,8 @@
    HBASE-1323  hbase-1234 broke TestThriftServer; fix and reenable
    HBASE-1425  ColumnValueFilter and WhileMatchFilter fixes on trunk
                (Clint Morgan via Stack)
+   HBASE-1431  NPE in HTable.checkAndSave when row doesn't exist (Guilherme
+               Mauro Germoglio Barbosa via Andrew Purtell)
 
   IMPROVEMENTS
    HBASE-1089  Add count of regions on filesystem to master UI; add percentage
@@ -254,6 +256,8 @@
                HLog#append?)
    HBASE-1429  Allow passing of a configuration object to HTablePool
    HBASE-1432  LuceneDocumentWrapper is not public
+   HBASE-1401  close HLog (and open new one) if there hasnt been edits in N
+               minutes/hours
 
   OPTIMIZATIONS
    HBASE-1412  Change values for delete column and column family in KeyValue

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=775816&r1=775815&r2=775816&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
(original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
Mon May 18 05:33:17 2009
@@ -592,6 +592,7 @@
           // instantiate the location
           location = new HRegionLocation(regionInfo,
               new HServerAddress(serverAddress));
+          LOG.debug(location);
           cacheLocation(tableName, location);
           return location;
         } catch (TableNotFoundException e) {

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java?rev=775816&r1=775815&r2=775816&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java
(original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java
Mon May 18 05:33:17 2009
@@ -998,6 +998,7 @@
     System.exit(0);
   }
 
+  @SuppressWarnings("null")
   protected static void doMain(String [] args,
       Class<? extends HMaster> masterClass) {
 

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java?rev=775816&r1=775815&r2=775816&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
(original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
Mon May 18 05:33:17 2009
@@ -132,7 +132,6 @@
   private final AtomicLong logSeqNum = new AtomicLong(0);
 
   private volatile long filenum = -1;
-  private volatile long old_filenum = -1;
   
   private final AtomicInteger numEntries = new AtomicInteger(0);
 
@@ -274,6 +273,10 @@
    * @throws IOException
    */
   public byte [] rollWriter() throws FailedLogCloseException, IOException {
+    // Return if nothing to flush.
+    if (this.writer != null && this.numEntries.get() <= 0) {
+      return null;
+    }
     byte [] regionToFlush = null;
     this.cacheFlushLock.lock();
     try {
@@ -283,9 +286,6 @@
       synchronized (updateLock) {
         // Clean up current writer.
         Path oldFile = cleanupCurrentWriter(this.filenum);
-        if (this.filenum >= 0) {
-          this.old_filenum = this.filenum;
-        }
         this.filenum = System.currentTimeMillis();
         Path newPath = computeFilename(this.filenum);
         this.writer = SequenceFile.createWriter(this.fs, this.conf, newPath,
@@ -587,7 +587,7 @@
       }
     }
   }
-  
+
   private void requestLogRoll() {
     if (this.listener != null) {
       this.listener.logRollRequested();

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=775816&r1=775815&r2=775816&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Mon May 18 05:33:17 2009
@@ -1396,13 +1396,17 @@
         Map<byte[],Cell> actualValues = getFull(row, keySet,
           HConstants.LATEST_TIMESTAMP, 1,lid);
         for (byte[] key : keySet) {
-          // If test fails exit
-          if(!Bytes.equals(actualValues.get(key).getValue(),
-              expectedValues.get(key))) {
-            success = false;
-            break;
-          }
-        }
+	  // If test fails exit
+	  Cell cell = actualValues.get(key);
+	  byte[] actualValue = new byte[] {};
+	  if (cell != null) 
+	    actualValue = cell.getValue();
+	  if(!Bytes.equals(actualValue,
+			   expectedValues.get(key))) {
+	    success = false;
+	    break;
+	  }
+	}
         if (success) {
           long commitTime = (b.getTimestamp() == LATEST_TIMESTAMP)?
             now: b.getTimestamp();

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/LogRoller.java?rev=775816&r1=775815&r2=775816&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
(original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
Mon May 18 05:33:17 2009
@@ -40,17 +40,32 @@
   private final ReentrantLock rollLock = new ReentrantLock();
   private final AtomicBoolean rollLog = new AtomicBoolean(false);
   private final HRegionServer server;
-  
+  private volatile long lastrolltime = System.currentTimeMillis();
+  // Period to roll log.
+  private final long rollperiod;
+
   /** @param server */
   public LogRoller(final HRegionServer server) {
     super();
     this.server = server;
+    this.rollperiod =
+      this.server.conf.getLong("hbase.regionserver.logroll.period", 3600000);
   }
 
   @Override
   public void run() {
     while (!server.isStopRequested()) {
+      long now = System.currentTimeMillis();
+      boolean periodic = false;
       if (!rollLog.get()) {
+        periodic = (now - this.lastrolltime) < this.rollperiod;
+        if (periodic) {
+          // Time for periodic roll
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Hlog roll period " + this.rollperiod + "ms elapsed");
+          }
+          break;
+        }
         synchronized (rollLog) {
           try {
             rollLog.wait(server.threadWakeFrequency);
@@ -62,6 +77,7 @@
       }
       rollLock.lock();          // Don't interrupt us. We're working
       try {
+        this.lastrolltime = now;
         byte [] regionToFlush = server.getLog().rollWriter();
         if (regionToFlush != null) {
           scheduleFlush(regionToFlush);
@@ -90,7 +106,7 @@
     }
     LOG.info("LogRoller exiting.");
   }
-  
+
   private void scheduleFlush(final byte [] region) {
     boolean scheduled = false;
     HRegion r = this.server.getOnlineRegion(region);
@@ -114,7 +130,7 @@
       rollLog.notifyAll();
     }
   }
-  
+
   /**
    * Called by region server to wake up this thread if it sleeping.
    * It is sleeping if rollLock is not held.
@@ -127,4 +143,4 @@
       rollLock.unlock();
     }
   }
-}
+}
\ No newline at end of file

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestHTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestHTable.java?rev=775816&r1=775815&r2=775816&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestHTable.java
(original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestHTable.java
Mon May 18 05:33:17 2009
@@ -235,12 +235,24 @@
     BatchUpdate batchUpdate = new BatchUpdate(row);
     BatchUpdate batchUpdate2 = new BatchUpdate(row);
     BatchUpdate batchUpdate3 = new BatchUpdate(row);
+
+    // this row doesn't exist when checkAndSave is invoked
+    byte [] row1 = Bytes.toBytes("row1");
+    BatchUpdate batchUpdate4 = new BatchUpdate(row1);
     
+    // to be used for a checkAndSave for expected empty columns
+    BatchUpdate batchUpdate5 = new BatchUpdate(row);
+
     HbaseMapWritable<byte[],byte[]> expectedValues =
       new HbaseMapWritable<byte[],byte[]>();
     HbaseMapWritable<byte[],byte[]> badExpectedValues =
       new HbaseMapWritable<byte[],byte[]>();
-    
+    HbaseMapWritable<byte[],byte[]> expectedNoValues =
+      new HbaseMapWritable<byte[],byte[]>();
+    // the columns used here must not be updated on batchupate
+    HbaseMapWritable<byte[],byte[]> expectedNoValues1 =
+      new HbaseMapWritable<byte[],byte[]>();
+
     for(int i = 0; i < 5; i++) {
       // This batchupdate is our initial batch update,
       // As such we also set our expected values to the same values
@@ -250,7 +262,12 @@
       
       badExpectedValues.put(Bytes.toBytes(COLUMN_FAMILY_STR+i),
         Bytes.toBytes(500));
-      
+
+      expectedNoValues.put(Bytes.toBytes(COLUMN_FAMILY_STR+i), new byte[] {});
+      // the columns used here must not be updated on batchupate
+      expectedNoValues1.put(Bytes.toBytes(COLUMN_FAMILY_STR+i+","+i), new byte[] {});
+
+
       // This is our second batchupdate that we will use to update the initial
       // batchupdate
       batchUpdate2.put(COLUMN_FAMILY_STR+i, Bytes.toBytes(i+1));
@@ -258,6 +275,14 @@
       // This final batch update is to check that our expected values (which
       // are now wrong)
       batchUpdate3.put(COLUMN_FAMILY_STR+i, Bytes.toBytes(i+2));
+
+      // Batch update that will not happen because it is to happen with some 
+      // expected values, but the row doesn't exist
+      batchUpdate4.put(COLUMN_FAMILY_STR+i, Bytes.toBytes(i));
+
+      // Batch update will happen: the row exists, but the expected columns don't,
+      // just as the condition
+      batchUpdate5.put(COLUMN_FAMILY_STR+i, Bytes.toBytes(i+3));
     }
     
     // Initialize rows
@@ -279,6 +304,58 @@
     
     // make sure that the old expected values fail
     assertFalse(table.checkAndSave(batchUpdate3, expectedValues,null));
+
+    // row doesn't exist, so doesn't matter the expected 
+    // values (unless they are empty) 
+    assertFalse(table.checkAndSave(batchUpdate4, badExpectedValues, null));
+
+    assertTrue(table.checkAndSave(batchUpdate4, expectedNoValues, null));
+    // make sure check and save saves the data when expected values were empty and the row
+    // didn't exist
+    r = table.getRow(row1);
+    columns = batchUpdate4.getColumns();
+    for(int i = 0; i < columns.length;i++) {
+      assertTrue(Bytes.equals(r.get(columns[i]).getValue(),batchUpdate4.get(columns[i])));
+    }  
+
+    // since the row isn't empty anymore, those expected (empty) values 
+    // are not valid anymore, so check and save method doesn't save. 
+    assertFalse(table.checkAndSave(batchUpdate4, expectedNoValues, null));
+    
+    // the row exists, but the columns don't. since the expected values are 
+    // for columns without value, checkAndSave must be successful. 
+    assertTrue(table.checkAndSave(batchUpdate5, expectedNoValues1, null));
+    // make sure checkAndSave saved values for batchUpdate5.
+    r = table.getRow(row);
+    columns = batchUpdate5.getColumns();
+    for(int i = 0; i < columns.length;i++) {
+      assertTrue(Bytes.equals(r.get(columns[i]).getValue(),batchUpdate5.get(columns[i])));
+    }  
+
+    // since the condition wasn't changed, the following checkAndSave 
+    // must also be successful.
+    assertTrue(table.checkAndSave(batchUpdate, expectedNoValues1, null));
+    // make sure checkAndSave saved values for batchUpdate1
+    r = table.getRow(row);
+    columns = batchUpdate.getColumns();
+    for(int i = 0; i < columns.length;i++) {
+      assertTrue(Bytes.equals(r.get(columns[i]).getValue(),batchUpdate.get(columns[i])));
+    }
+
+    // one failing condition must make the following checkAndSave fail
+    // the failing condition is a column to be empty, however, it has a value.
+    HbaseMapWritable<byte[],byte[]> expectedValues1 =
+      new HbaseMapWritable<byte[],byte[]>();
+    expectedValues1.put(Bytes.toBytes(COLUMN_FAMILY_STR+0), new byte[] {});
+    expectedValues1.put(Bytes.toBytes(COLUMN_FAMILY_STR+"EMPTY+ROW"), new byte[] {});
+    assertFalse(table.checkAndSave(batchUpdate5, expectedValues1, null));
+
+    // assure the values on the row remain the same
+    r = table.getRow(row);
+    columns = batchUpdate.getColumns();
+    for(int i = 0; i < columns.length;i++) {
+      assertTrue(Bytes.equals(r.get(columns[i]).getValue(),batchUpdate.get(columns[i])));
+    }    
   }
 
   /**
@@ -373,5 +450,5 @@
      fail("Should have thrown a TableNotFoundException instead of a " +
        e.getClass());
    }
-  }
+   }
 }



Mime
View raw message