incubator-accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1239912 - in /incubator/accumulo/branches/1.4: src/core/src/main/java/org/apache/accumulo/core/data/ src/server/src/main/java/org/apache/accumulo/server/master/state/ src/server/src/main/java/org/apache/accumulo/server/master/tableOps/ tes...
Date Thu, 02 Feb 2012 23:03:13 GMT
Author: kturner
Date: Thu Feb  2 23:03:12 2012
New Revision: 1239912

URL: http://svn.apache.org/viewvc?rev=1239912&view=rev
Log:
ACCUMULO-315 ACCUMULO-366 
 * made merge check first tablet during verification
 * made delete table check prev row when verifying tablets offline
 * added merge functional test that does exhaustive test of split points vs merge points

Modified:
    incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
    incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
    incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java
    incubator/accumulo/branches/1.4/test/system/auto/simple/merge.py

Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java?rev=1239912&r1=1239911&r2=1239912&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
(original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
Thu Feb  2 23:03:12 2012
@@ -754,4 +754,24 @@ public class KeyExtent implements Writab
     return new TKeyExtent(TextUtil.getByteBuffer(textTableId), textEndRow == null ? null
: TextUtil.getByteBuffer(textEndRow), textPrevEndRow == null ? null
         : TextUtil.getByteBuffer(textPrevEndRow));
   }
+  
+  /**
+   * @param prevExtent
+   */
+  public boolean isPreviousExtent(KeyExtent prevExtent) {
+    if (prevExtent == null)
+      return getPrevEndRow() == null;
+    
+    if (!prevExtent.getTableId().equals(getTableId()))
+      throw new IllegalArgumentException("Cannot compare accross tables " + prevExtent +
" " + this);
+    
+    if (prevExtent.getEndRow() == null)
+      return false;
+    
+    if (getPrevEndRow() == null)
+      return false;
+    
+    return prevExtent.getEndRow().equals(getPrevEndRow());
+    
+  }
 }

Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java?rev=1239912&r1=1239911&r2=1239912&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
(original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
Thu Feb  2 23:03:12 2012
@@ -169,19 +169,32 @@ public class MergeStats {
     }
     Text tableId = extent.getTableId();
     Text first = KeyExtent.getMetadataEntry(tableId, start);
-    Range range = new Range(first, true, null, true);
+    Range range = new Range(first, false, null, true);
     scanner.setRange(range);
-    Text pr = null;
+    KeyExtent prevExtent = null;
+
     for (Entry<Key,Value> entry : scanner) {
       TabletLocationState tls = MetaDataTableScanner.createTabletLocationState(entry.getKey(),
entry.getValue());
       if (!tls.extent.getTableId().equals(tableId)) {
         break;
       }
-      verify.update(tls.extent, tls.getState(master.onlineTabletServers()), tls.chopped);
-      // check that the prevRow matches the previous row
-      if (pr != null && (tls.extent.getPrevEndRow() == null || !tls.extent.getPrevEndRow().equals(pr)))
+      
+      if (prevExtent == null) {
+        // this is the first tablet observed, it must be offline and its prev row must be
less than the start of the merge range
+        if (tls.extent.getPrevEndRow() != null && tls.extent.getPrevEndRow().compareTo(start)
> 0) {
+          return false;
+        }
+        
+        if (tls.getState(master.onlineTabletServers()) != TabletState.UNASSIGNED)
+          return false;
+        
+      } else if (!tls.extent.isPreviousExtent(prevExtent)) {
         return false;
-      pr = tls.extent.getEndRow();
+      }
+      
+      prevExtent = tls.extent;
+
+      verify.update(tls.extent, tls.getState(master.onlineTabletServers()), tls.chopped);
       // stop when we've seen the tablet just beyond our range
       if (tls.extent.getPrevEndRow() != null && extent.getEndRow() != null &&
tls.extent.getPrevEndRow().compareTo(extent.getEndRow()) > 0) {
         break;

Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java?rev=1239912&r1=1239911&r2=1239912&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java
(original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java
Thu Feb  2 23:03:12 2012
@@ -26,6 +26,7 @@ import org.apache.accumulo.core.client.B
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.impl.Tables;
 import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 import org.apache.accumulo.core.data.Key;
@@ -89,21 +90,29 @@ class CleanUp extends MasterRepo {
     
     boolean done = true;
     Range tableRange = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
-    MetaDataTableScanner metaDataTableScanner = new MetaDataTableScanner(environment.getInstance(),
SecurityConstants.getSystemCredentials(), tableRange, null);
-    try {
-      while (metaDataTableScanner.hasNext()) {
-        TabletLocationState locationState = metaDataTableScanner.next();
-        TabletState state = locationState.getState(environment.onlineTabletServers());
-        if (state.equals(TabletState.ASSIGNED) || state.equals(TabletState.HOSTED)) {
-          log.debug("Still waiting for table to be deleted: " + tableId + " locationState:
" + locationState);
-          done = false;
-          break;
-        }
+    Scanner scanner = environment.getInstance().getConnector(SecurityConstants.getSystemCredentials())
+        .createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+    MetaDataTableScanner.configureScanner(scanner, environment);
+    scanner.setRange(tableRange);
+    
+    KeyExtent prevExtent = null;
+    for (Entry<Key,Value> entry : scanner) {
+      TabletLocationState locationState = MetaDataTableScanner.createTabletLocationState(entry.getKey(),
entry.getValue());
+      if (!locationState.extent.isPreviousExtent(prevExtent)) {
+        log.debug("Still waiting for table to be deleted: " + tableId + " saw inconsistency"
+ prevExtent + " " + locationState.extent);
+        done = false;
+        break;
+      }
+      prevExtent = locationState.extent;
+      
+      TabletState state = locationState.getState(environment.onlineTabletServers());
+      if (state.equals(TabletState.ASSIGNED) || state.equals(TabletState.HOSTED)) {
+        log.debug("Still waiting for table to be deleted: " + tableId + " locationState:
" + locationState);
+        done = false;
+        break;
       }
-    } finally {
-      metaDataTableScanner.close();
     }
-    
+
     if (!done)
       return 50;
     

Modified: incubator/accumulo/branches/1.4/test/system/auto/simple/merge.py
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/test/system/auto/simple/merge.py?rev=1239912&r1=1239911&r2=1239912&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/test/system/auto/simple/merge.py (original)
+++ incubator/accumulo/branches/1.4/test/system/auto/simple/merge.py Thu Feb  2 23:03:12 2012
@@ -18,6 +18,7 @@ import logging
 import unittest
 
 from TestUtils import TestUtilsMixin
+from JavaTest import JavaTest
 
 log = logging.getLogger('test.auto')
 
@@ -90,9 +91,16 @@ quit
         secondMerge = secondMerge.strip().split('\n')[:-1]
         self.assert_(secondMerge == ['c','e','f','y'])
 
+class MergeTest(JavaTest):
+    "Test Merge"
+
+    order = 92
+    testClass="org.apache.accumulo.server.test.functional.MergeTest"
+
 
 def suite():
     result = unittest.TestSuite()
     result.addTest(Merge())
     result.addTest(MergeSize())
+    result.addTest(MergeTest())
     return result



Mime
View raw message