accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1466211 - in /accumulo/branches/1.5: core/src/main/java/org/apache/accumulo/core/util/ server/src/main/java/org/apache/accumulo/server/tabletserver/ server/src/main/java/org/apache/accumulo/server/util/ server/src/test/java/org/apache/accu...
Date Tue, 09 Apr 2013 20:21:57 GMT
Author: kturner
Date: Tue Apr  9 20:21:57 2013
New Revision: 1466211

URL: http://svn.apache.org/r1466211
Log:
ACCUMULO-1243 Made tablet loading code only load one tablet when recovering a split.  Made code more strict, it will only load an exact tablet.  Made load code roll back splits that have started, but did not create a new tablet.  Added some more test.   

Added:
    accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/tabletserver/CheckTabletMetadataTest.java
    accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/TestAccumuloSplitRecovery.java
      - copied, changed from r1466018, accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/TestAccumulo1235.java
Removed:
    accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/TestAccumulo1235.java
Modified:
    accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java
    accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
    accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
    accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java
    accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java

Modified: accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java?rev=1466211&r1=1466210&r2=1466211&view=diff
==============================================================================
--- accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java (original)
+++ accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java Tue Apr  9 20:21:57 2013
@@ -32,7 +32,6 @@ import org.apache.accumulo.core.client.A
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.impl.ScannerImpl;
 import org.apache.accumulo.core.client.impl.Tables;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.KeyExtent;
@@ -173,13 +172,7 @@ public class MetadataTable {
     
     return new Pair<SortedMap<KeyExtent,Text>,List<KeyExtent>>(results, locationless);
   }
-  
-  public static SortedMap<Text,SortedMap<ColumnFQ,Value>> getTabletEntries(Instance instance, KeyExtent ke, List<ColumnFQ> columns, TCredentials credentials) {
-    TreeMap<Key,Value> tkv = new TreeMap<Key,Value>();
-    getTabletAndPrevTabletKeyValues(instance, tkv, ke, columns, credentials);
-    return getTabletEntries(tkv, columns);
-  }
-  
+
   public static SortedMap<Text,SortedMap<ColumnFQ,Value>> getTabletEntries(SortedMap<Key,Value> tabletKeyValues, List<ColumnFQ> columns) {
     TreeMap<Text,SortedMap<ColumnFQ,Value>> tabletEntries = new TreeMap<Text,SortedMap<ColumnFQ,Value>>();
     
@@ -207,40 +200,7 @@ public class MetadataTable {
     
     return tabletEntries;
   }
-  
-  public static void getTabletAndPrevTabletKeyValues(Instance instance, SortedMap<Key,Value> tkv, KeyExtent ke, List<ColumnFQ> columns, TCredentials credentials) {
-    Text startRow;
-    Text endRow = ke.getMetadataEntry();
     
-    if (ke.getPrevEndRow() == null) {
-      startRow = new Text(KeyExtent.getMetadataEntry(ke.getTableId(), new Text()));
-    } else {
-      startRow = new Text(KeyExtent.getMetadataEntry(ke.getTableId(), ke.getPrevEndRow()));
-    }
-    
-    Scanner scanner = new ScannerImpl(instance, credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
-    
-    if (columns != null) {
-      for (ColumnFQ column : columns)
-        column.fetch(scanner);
-    }
-    
-    scanner.setRange(new Range(new Key(startRow), true, new Key(endRow).followingKey(PartialKey.ROW), false));
-    
-    tkv.clear();
-    boolean successful = false;
-    try {
-      for (Entry<Key,Value> entry : scanner) {
-        tkv.put(entry.getKey(), entry.getValue());
-      }
-      successful = true;
-    } finally {
-      if (!successful) {
-        tkv.clear();
-      }
-    }
-  }
-  
   public static void getEntries(Instance instance, TCredentials credentials, String table, boolean isTid, Map<KeyExtent,String> locations,
       SortedSet<KeyExtent> tablets) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
     String tableId = isTid ? table : Tables.getNameToIdMap(instance).get(table);

Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1466211&r1=1466210&r2=1466211&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java (original)
+++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java Tue Apr  9 20:21:57 2013
@@ -68,6 +68,7 @@ import org.apache.accumulo.core.Constant
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
 import org.apache.accumulo.core.client.impl.TabletType;
 import org.apache.accumulo.core.client.impl.Translator;
 import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
@@ -2442,10 +2443,10 @@ public class TabletServer extends Abstra
       log.debug("Loading extent: " + extent);
       
       // check Metadata table before accepting assignment
-      SortedMap<KeyExtent,Text> tabletsInRange = null;
+      Text locationToOpen = null;
       SortedMap<Key,Value> tabletsKeyValues = new TreeMap<Key,Value>();
       try {
-        tabletsInRange = verifyTabletInformation(extent, TabletServer.this.getTabletSession(), tabletsKeyValues, getClientAddressString(), getLock());
+        locationToOpen = verifyTabletInformation(extent, TabletServer.this.getTabletSession(), tabletsKeyValues, getClientAddressString(), getLock());
       } catch (Exception e) {
         synchronized (openingTablets) {
           openingTablets.remove(extent);
@@ -2454,131 +2455,92 @@ public class TabletServer extends Abstra
         log.warn("Failed to verify tablet " + extent, e);
         throw new RuntimeException(e);
       }
-      
-      if (tabletsInRange == null) {
-        log.info("Reporting tablet " + extent + " assignment failure: unable to verify Tablet Information");
+
+      if (locationToOpen == null) {
+        log.debug("Reporting tablet " + extent + " assignment failure: unable to verify Tablet Information");
         enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.LOAD_FAILURE, extent));
         synchronized (openingTablets) {
           openingTablets.remove(extent);
           openingTablets.notifyAll();
         }
+        
         return;
       }
-      // If extent given is not the one to be opened, update
-      if (tabletsInRange.size() != 1 || !tabletsInRange.containsKey(extent)) {
-        synchronized (openingTablets) {
-          openingTablets.remove(extent);
-          openingTablets.notifyAll();
-          for (KeyExtent e : tabletsInRange.keySet())
-            openingTablets.add(e);
+
+      Tablet tablet = null;
+      boolean successful = false;
+      
+      try {
+        TabletResourceManager trm = resourceManager.createTabletResourceManager();
+        
+        // this opens the tablet file and fills in the endKey in the
+        // extent
+        tablet = new Tablet(TabletServer.this, locationToOpen, extent, trm, tabletsKeyValues);
+        /*
+         * If a minor compaction starts after a tablet opens, this indicates a log recovery occurred. This recovered data must be minor compacted.
+         * 
+         * There are three reasons to wait for this minor compaction to finish before placing the tablet in online tablets.
+         * 
+         * 1) The log recovery code does not handle data written to the tablet on multiple tablet servers. 2) The log recovery code does not block if memory is
+         * full. Therefore recovering lots of tablets that use a lot of memory could run out of memory. 3) The minor compaction finish event did not make it to
+         * the logs (the file will be in !METADATA, preventing replay of compacted data)... but do not want a majc to wipe the file out from !METADATA and then
+         * have another process failure... this could cause duplicate data to replay
+         */
+        if (tablet.getNumEntriesInMemory() > 0 && !tablet.minorCompactNow(MinorCompactionReason.SYSTEM)) {
+          throw new RuntimeException("Minor compaction after recovery fails for " + extent);
         }
-      } else {
-        // remove any metadata entries for the previous tablet
-        Iterator<Key> iter = tabletsKeyValues.keySet().iterator();
-        Text row = extent.getMetadataEntry();
-        while (iter.hasNext()) {
-          Key key = iter.next();
-          if (!key.getRow().equals(row)) {
-            iter.remove();
+
+        Assignment assignment = new Assignment(extent, getTabletSession());
+        TabletStateStore.setLocation(assignment);
+        
+        synchronized (openingTablets) {
+          synchronized (onlineTablets) {
+            openingTablets.remove(extent);
+            onlineTablets.put(extent, tablet);
+            openingTablets.notifyAll();
+            recentlyUnloadedCache.remove(tablet);
           }
         }
+        tablet = null; // release this reference
+        successful = true;
+      } catch (Throwable e) {
+        log.warn("exception trying to assign tablet " + extent + " " + locationToOpen, e);
+        if (e.getMessage() != null)
+          log.warn(e.getMessage());
+        String table = extent.getTableId().toString();
+        ProblemReports.getInstance().report(new ProblemReport(table, TABLET_LOAD, extent.getUUID().toString(), getClientAddressString(), e));
       }
-      if (tabletsInRange.size() > 1) {
-        log.debug("Master didn't know " + extent + " was split, letting it know about " + tabletsInRange.keySet());
-        enqueueMasterMessage(new SplitReportMessage(extent, tabletsInRange));
-      }
-      
-      // create the tablet object
-      for (Entry<KeyExtent,Text> entry : tabletsInRange.entrySet()) {
-        Tablet tablet = null;
-        boolean successful = false;
-        
-        final KeyExtent extentToOpen = entry.getKey();
-        Text locationToOpen = entry.getValue();
-        
-        if (onlineTablets.containsKey(extentToOpen)) {
-          // know this was from fixing a split, because initial check
-          // would have caught original extent
-          log.warn("Something is screwy!  Already serving tablet " + extentToOpen + " derived from fixing split. Original extent = " + extent);
+      
+      if (!successful) {
+        synchronized (unopenedTablets) {
           synchronized (openingTablets) {
-            openingTablets.remove(extentToOpen);
+            openingTablets.remove(extent);
+            unopenedTablets.add(extent);
             openingTablets.notifyAll();
           }
-          continue;
         }
-        
-        try {
-          TabletResourceManager trm = resourceManager.createTabletResourceManager();
-          
-          // this opens the tablet file and fills in the endKey in the
-          // extent
-          tablet = new Tablet(TabletServer.this, locationToOpen, extentToOpen, trm, tabletsKeyValues);
-          /*
-           * If a minor compaction starts after a tablet opens, this indicates a log recovery occurred. This recovered data must be minor compacted.
-           * 
-           * There are three reasons to wait for this minor compaction to finish before placing the tablet in online tablets.
-           * 
-           * 1) The log recovery code does not handle data written to the tablet on multiple tablet servers. 2) The log recovery code does not block if memory
-           * is full. Therefore recovering lots of tablets that use a lot of memory could run out of memory. 3) The minor compaction finish event did not make
-           * it to the logs (the file will be in !METADATA, preventing replay of compacted data)... but do not want a majc to wipe the file out from !METADATA
-           * and then have another process failure... this could cause duplicate data to replay
-           */
-          if (tablet.getNumEntriesInMemory() > 0 && !tablet.minorCompactNow(MinorCompactionReason.SYSTEM)) {
-            throw new RuntimeException("Minor compaction after recovery fails for " + extentToOpen);
-          }
-          
-          Assignment assignment = new Assignment(extentToOpen, getTabletSession());
-          TabletStateStore.setLocation(assignment);
-          
-          synchronized (openingTablets) {
-            synchronized (onlineTablets) {
-              openingTablets.remove(extentToOpen);
-              onlineTablets.put(extentToOpen, tablet);
-              openingTablets.notifyAll();
-              recentlyUnloadedCache.remove(tablet);
-            }
-          }
-          tablet = null; // release this reference
-          successful = true;
-        } catch (Throwable e) {
-          log.warn("exception trying to assign tablet " + extentToOpen + " " + locationToOpen, e);
-          if (e.getMessage() != null)
-            log.warn(e.getMessage());
-          String table = extent.getTableId().toString();
-          ProblemReports.getInstance().report(new ProblemReport(table, TABLET_LOAD, extentToOpen.getUUID().toString(), getClientAddressString(), e));
-        }
-        
-        if (!successful) {
-          synchronized (unopenedTablets) {
-            synchronized (openingTablets) {
-              openingTablets.remove(extentToOpen);
-              unopenedTablets.add(extentToOpen);
-              openingTablets.notifyAll();
-            }
-          }
-          log.warn("failed to open tablet " + extentToOpen + " reporting failure to master");
-          enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.LOAD_FAILURE, extentToOpen));
-          long reschedule = Math.min((1l << Math.min(32, retryAttempt)) * 1000, 10 * 60 * 1000l);
-          log.warn(String.format("rescheduling tablet load in %.2f seconds", reschedule / 1000.));
-          SimpleTimer.getInstance().schedule(new TimerTask() {
-            @Override
-            public void run() {
-              log.info("adding tablet " + extent + " back to the assignment pool (retry " + retryAttempt + ")");
-              AssignmentHandler handler = new AssignmentHandler(extentToOpen, retryAttempt + 1);
-              if (extent.isMeta()) {
-                if (extent.isRootTablet()) {
-                  new Daemon(new LoggingRunnable(log, handler), "Root tablet assignment retry").start();
-                } else {
-                  resourceManager.addMetaDataAssignment(handler);
-                }
+        log.warn("failed to open tablet " + extent + " reporting failure to master");
+        enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.LOAD_FAILURE, extent));
+        long reschedule = Math.min((1l << Math.min(32, retryAttempt)) * 1000, 10 * 60 * 1000l);
+        log.warn(String.format("rescheduling tablet load in %.2f seconds", reschedule / 1000.));
+        SimpleTimer.getInstance().schedule(new TimerTask() {
+          @Override
+          public void run() {
+            log.info("adding tablet " + extent + " back to the assignment pool (retry " + retryAttempt + ")");
+            AssignmentHandler handler = new AssignmentHandler(extent, retryAttempt + 1);
+            if (extent.isMeta()) {
+              if (extent.isRootTablet()) {
+                new Daemon(new LoggingRunnable(log, handler), "Root tablet assignment retry").start();
               } else {
-                resourceManager.addAssignment(handler);
+                resourceManager.addMetaDataAssignment(handler);
               }
+            } else {
+              resourceManager.addAssignment(handler);
             }
-          }, reschedule);
-        } else {
-          enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.LOADED, extentToOpen));
-        }
+          }
+        }, reschedule);
+      } else {
+        enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.LOADED, extent));
       }
     }
   }
@@ -2890,148 +2852,141 @@ public class TabletServer extends Abstra
   
   private long totalMinorCompactions;
   
-  public static SortedMap<KeyExtent,Text> verifyTabletInformation(KeyExtent extent, TServerInstance instance, SortedMap<Key,Value> tabletsKeyValues,
-      String clientAddress, ZooLock lock) throws AccumuloSecurityException, DistributedStoreException {
-    for (int tries = 0; tries < 3; tries++) {
-      try {
-        log.debug("verifying extent " + extent);
-        if (extent.isRootTablet()) {
-          ZooTabletStateStore store = new ZooTabletStateStore();
-          if (!store.iterator().hasNext()) {
-            log.warn("Illegal state: location is not set in zookeeper");
-            return null;
-          }
-          TabletLocationState next = store.iterator().next();
-          if (!instance.equals(next.future)) {
-            log.warn("Future location is not to this server for the root tablet");
-            return null;
-          }
-          TreeMap<KeyExtent,Text> set = new TreeMap<KeyExtent,Text>();
-          set.put(extent, new Text(Constants.ZROOT_TABLET));
-          return set;
-        }
-        
-        List<ColumnFQ> columnsToFetch = Arrays.asList(new ColumnFQ[] {Constants.METADATA_DIRECTORY_COLUMN, Constants.METADATA_PREV_ROW_COLUMN,
-            Constants.METADATA_SPLIT_RATIO_COLUMN, Constants.METADATA_OLD_PREV_ROW_COLUMN, Constants.METADATA_TIME_COLUMN});
-        
-        if (tabletsKeyValues == null) {
-          tabletsKeyValues = new TreeMap<Key,Value>();
-        }
-        MetadataTable.getTabletAndPrevTabletKeyValues(tabletsKeyValues, extent, null, SecurityConstants.getSystemCredentials());
-        
-        SortedMap<Text,SortedMap<ColumnFQ,Value>> tabletEntries;
-        tabletEntries = MetadataTable.getTabletEntries(tabletsKeyValues, columnsToFetch);
-        
-        if (tabletEntries.size() == 0) {
-          log.warn("Failed to find any metadata entries for " + extent);
-          return null;
-        }
-        
-        // ensure last key in map is same as extent that was passed in
-        if (!tabletEntries.lastKey().equals(extent.getMetadataEntry())) {
-          log.warn("Failed to find metadata entry for " + extent + " found " + tabletEntries.lastKey());
-          return null;
-        }
-        
-        TServerInstance future = null;
-        Text metadataEntry = extent.getMetadataEntry();
-        for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
-          Key key = entry.getKey();
-          if (!metadataEntry.equals(key.getRow()))
-            continue;
-          Text cf = key.getColumnFamily();
-          if (cf.equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY)) {
-            future = new TServerInstance(entry.getValue(), key.getColumnQualifier());
-          } else if (cf.equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)) {
-            log.error("Tablet seems to be already assigned to " + new TServerInstance(entry.getValue(), key.getColumnQualifier()));
-            return null;
-          }
-        }
-        if (future == null) {
-          log.warn("The master has not assigned " + extent + " to " + instance);
-          return null;
-        }
-        if (!instance.equals(future)) {
-          log.warn("Table " + extent + " has been assigned to " + future + " which is not " + instance);
-          return null;
-        }
-        
-        // look for incomplete splits
-        int splitsFixed = 0;
-        for (Entry<Text,SortedMap<ColumnFQ,Value>> entry : tabletEntries.entrySet()) {
-          
-          if (extent.getPrevEndRow() != null) {
-            Text prevRowMetadataEntry = new Text(KeyExtent.getMetadataEntry(extent.getTableId(), extent.getPrevEndRow()));
-            if (entry.getKey().compareTo(prevRowMetadataEntry) <= 0) {
-              continue;
-            }
-          }
-          
-          if (entry.getValue().containsKey(Constants.METADATA_OLD_PREV_ROW_COLUMN)) {
-            KeyExtent fixedke = MetadataTable.fixSplit(entry.getKey(), entry.getValue(), instance, SecurityConstants.getSystemCredentials(), lock);
-            if (fixedke != null) {
-              if (fixedke.getPrevEndRow() == null || fixedke.getPrevEndRow().compareTo(extent.getPrevEndRow()) < 0) {
-                extent = new KeyExtent(extent);
-                extent.setPrevEndRow(fixedke.getPrevEndRow());
-              }
-              splitsFixed++;
-            }
-          }
-        }
-        
-        if (splitsFixed > 0) {
-          // reread and reverify metadata entries now that metadata
-          // entries were fixed
-          tabletsKeyValues.clear();
-          return verifyTabletInformation(extent, instance, tabletsKeyValues, clientAddress, lock);
-        }
-        
-        SortedMap<KeyExtent,Text> children = new TreeMap<KeyExtent,Text>();
-        
-        for (Entry<Text,SortedMap<ColumnFQ,Value>> entry : tabletEntries.entrySet()) {
-          if (extent.getPrevEndRow() != null) {
-            Text prevRowMetadataEntry = new Text(KeyExtent.getMetadataEntry(extent.getTableId(), extent.getPrevEndRow()));
-            
-            if (entry.getKey().compareTo(prevRowMetadataEntry) <= 0) {
-              continue;
-            }
-          }
-          
-          Value prevEndRowIBW = entry.getValue().get(Constants.METADATA_PREV_ROW_COLUMN);
-          if (prevEndRowIBW == null) {
-            log.warn("Metadata entry does not have prev row (" + entry.getKey() + ")");
-            return null;
-          }
-          
-          Value dirIBW = entry.getValue().get(Constants.METADATA_DIRECTORY_COLUMN);
-          if (dirIBW == null) {
-            log.warn("Metadata entry does not have directory (" + entry.getKey() + ")");
-            return null;
-          }
-          
-          Text dir = new Text(dirIBW.get());
-          
-          KeyExtent child = new KeyExtent(entry.getKey(), prevEndRowIBW);
-          children.put(child, dir);
-        }
-        
-        if (!MetadataTable.isContiguousRange(extent, new TreeSet<KeyExtent>(children.keySet()))) {
-          log.warn("For extent " + extent + " metadata entries " + children + " do not form a contiguous range.");
-          return null;
-        }
-        return children;
-      } catch (AccumuloException e) {
-        log.error("error verifying metadata information. retrying ...");
-        log.error(e.toString());
-        UtilWaitThread.sleep(1000);
-      } catch (AccumuloSecurityException e) {
-        // if it's a security exception, retrying won't work either.
-        log.error(e.toString());
-        throw e;
+  private static Text verifyRootTablet(KeyExtent extent, TServerInstance instance) throws DistributedStoreException, AccumuloException {
+    ZooTabletStateStore store = new ZooTabletStateStore();
+    if (!store.iterator().hasNext()) {
+      throw new AccumuloException("Illegal state: location is not set in zookeeper");
+    }
+    TabletLocationState next = store.iterator().next();
+    if (!instance.equals(next.future)) {
+      throw new AccumuloException("Future location is not to this server for the root tablet");
+    }
+    
+    if (next.current != null) {
+      throw new AccumuloException("Root tablet already has a location set");
+    }
+    
+    return new Text(Constants.ZROOT_TABLET);
+  }
+  
+  public static Text verifyTabletInformation(KeyExtent extent, TServerInstance instance, SortedMap<Key,Value> tabletsKeyValues, String clientAddress,
+      ZooLock lock) throws AccumuloSecurityException, DistributedStoreException, AccumuloException {
+    
+    log.debug("verifying extent " + extent);
+    if (extent.isRootTablet()) {
+      return verifyRootTablet(extent, instance);
+    }
+    
+    List<ColumnFQ> columnsToFetch = Arrays.asList(new ColumnFQ[] {Constants.METADATA_DIRECTORY_COLUMN, Constants.METADATA_PREV_ROW_COLUMN,
+        Constants.METADATA_SPLIT_RATIO_COLUMN, Constants.METADATA_OLD_PREV_ROW_COLUMN, Constants.METADATA_TIME_COLUMN});
+    
+    ScannerImpl scanner = new ScannerImpl(HdfsZooInstance.getInstance(), SecurityConstants.getSystemCredentials(), Constants.METADATA_TABLE_ID,
+        Constants.NO_AUTHS);
+    scanner.setRange(extent.toMetadataRange());
+
+    TreeMap<Key,Value> tkv = new TreeMap<Key,Value>();
+    for (Entry<Key,Value> entry : scanner)
+      tkv.put(entry.getKey(), entry.getValue());
+
+    // only populate map after success
+    if (tabletsKeyValues == null) {
+      tabletsKeyValues = tkv;
+    } else {
+      tabletsKeyValues.clear();
+      tabletsKeyValues.putAll(tkv);
+    }
+    
+    Text metadataEntry = extent.getMetadataEntry();
+    
+    Value dir = checkTabletMetadata(extent, instance, tabletsKeyValues, metadataEntry);
+    if (dir == null)
+      return null;
+    
+    Value oldPrevEndRow = null;
+    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+      if (Constants.METADATA_OLD_PREV_ROW_COLUMN.hasColumns(entry.getKey())) {
+        oldPrevEndRow = entry.getValue();
       }
     }
-    // default is to accept
-    return null;
+    
+    if (oldPrevEndRow != null) {
+      SortedMap<Text,SortedMap<ColumnFQ,Value>> tabletEntries;
+      tabletEntries = MetadataTable.getTabletEntries(tabletsKeyValues, columnsToFetch);
+      
+      KeyExtent fke = MetadataTable.fixSplit(metadataEntry, tabletEntries.get(metadataEntry), instance, SecurityConstants.getSystemCredentials(), lock);
+      
+      if (!fke.equals(extent)) {
+        return null;
+      }
+      
+      // reread and reverify metadata entries now that metadata entries were fixed
+      tabletsKeyValues.clear();
+      return verifyTabletInformation(fke, instance, tabletsKeyValues, clientAddress, lock);
+    }
+    
+    return new Text(dir.get());
+  }
+  
+  static Value checkTabletMetadata(KeyExtent extent, TServerInstance instance, SortedMap<Key,Value> tabletsKeyValues, Text metadataEntry)
+      throws AccumuloException {
+    
+    TServerInstance future = null;
+    Value prevEndRow = null;
+    Value dir = null;
+    Value time = null;
+    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+      Key key = entry.getKey();
+      if (!metadataEntry.equals(key.getRow())) {
+        log.info("Unexpected row in tablet metadata " + metadataEntry + " " + key.getRow());
+        return null;
+      }
+      Text cf = key.getColumnFamily();
+      if (cf.equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY)) {
+        if (future != null) {
+          throw new AccumuloException("Tablet has multiple future locations " + extent);
+        }
+        future = new TServerInstance(entry.getValue(), key.getColumnQualifier());
+      } else if (cf.equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)) {
+        log.info("Tablet seems to be already assigned to " + new TServerInstance(entry.getValue(), key.getColumnQualifier()));
+        return null;
+      } else if (Constants.METADATA_PREV_ROW_COLUMN.hasColumns(key)) {
+        prevEndRow = entry.getValue();
+      } else if (Constants.METADATA_DIRECTORY_COLUMN.hasColumns(key)) {
+        dir = entry.getValue();
+      } else if (Constants.METADATA_TIME_COLUMN.hasColumns(key)) {
+        time = entry.getValue();
+      }
+    }
+    
+    if (prevEndRow == null) {
+      throw new AccumuloException("Metadata entry does not have prev row (" + metadataEntry + ")");
+    } else {
+      KeyExtent ke2 = new KeyExtent(metadataEntry, prevEndRow);
+      if (!extent.equals(ke2)) {
+        log.info("Tablet prev end row mismatch " + extent + " " + ke2.getPrevEndRow());
+        return null;
+      }
+    }
+
+    if (dir == null) {
+      throw new AccumuloException("Metadata entry does not have directory (" + metadataEntry + ")");
+    }
+
+    if (time == null) {
+      throw new AccumuloException("Metadata entry does not have time (" + metadataEntry + ")");
+    }
+
+    if (future == null) {
+      log.info("The master has not assigned " + extent + " to " + instance);
+      return null;
+    }
+
+    if (!instance.equals(future)) {
+      log.info("Table " + extent + " has been assigned to " + future + " which is not " + instance);
+      return null;
+    }
+    
+    return dir;
   }
   
   public String getClientAddressString() {

Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java?rev=1466211&r1=1466210&r2=1466211&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java (original)
+++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java Tue Apr  9 20:21:57 2013
@@ -339,7 +339,6 @@ public class MetadataTable extends org.a
     TreeMap<String,DataFileValue> sizes = new TreeMap<String,DataFileValue>();
     
     Scanner mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
-    mdScanner.setRange(Constants.METADATA_KEYSPACE);
     mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
     Text row = extent.getMetadataEntry();
     
@@ -386,6 +385,14 @@ public class MetadataTable extends org.a
     update(credentials, zooLock, m);
   }
   
+  public static void rollBackSplit(Text metadataEntry, Text oldPrevEndRow, TCredentials credentials, ZooLock zooLock) {
+    KeyExtent ke = new KeyExtent(metadataEntry, oldPrevEndRow);
+    Mutation m = ke.getPrevRowUpdateMutation();
+    Constants.METADATA_SPLIT_RATIO_COLUMN.putDelete(m);
+    Constants.METADATA_OLD_PREV_ROW_COLUMN.putDelete(m);
+    update(credentials, zooLock, m);
+  }
+
   public static void splitTablet(KeyExtent extent, Text oldPrevEndRow, double splitRatio, TCredentials credentials, ZooLock zooLock) {
     Mutation m = extent.getPrevRowUpdateMutation(); //
     
@@ -490,14 +497,6 @@ public class MetadataTable extends org.a
     update(credentials, zooLock, m);
   }
   
-  public static void getTabletAndPrevTabletKeyValues(SortedMap<Key,Value> tkv, KeyExtent ke, List<ColumnFQ> columns, TCredentials credentials) {
-    getTabletAndPrevTabletKeyValues(HdfsZooInstance.getInstance(), tkv, ke, columns, credentials);
-  }
-  
-  public static SortedMap<Text,SortedMap<ColumnFQ,Value>> getTabletEntries(KeyExtent ke, List<ColumnFQ> columns, TCredentials credentials) {
-    return getTabletEntries(HdfsZooInstance.getInstance(), ke, columns, credentials);
-  }
-  
   private static KeyExtent fixSplit(Text table, Text metadataEntry, Text metadataPrevEndRow, Value oper, double splitRatio, TServerInstance tserver,
       TCredentials credentials, String time, long initFlushID, long initCompactID, ZooLock lock) throws AccumuloException {
     if (metadataPrevEndRow == null)
@@ -505,51 +504,45 @@ public class MetadataTable extends org.a
       // prev end row....
       throw new AccumuloException("Split tablet does not have prev end row, something is amiss, extent = " + metadataEntry);
     
-    KeyExtent low = null;
-    
-    List<String> highDatafilesToRemove = new ArrayList<String>();
-    
-    String lowDirectory = TabletOperations.createTabletDirectory(ServerConstants.getTablesDir() + "/" + table, metadataPrevEndRow);
-    
-    Text prevPrevEndRow = KeyExtent.decodePrevEndRow(oper);
-    
-    low = new KeyExtent(table, metadataPrevEndRow, prevPrevEndRow);
-    
-    Scanner scanner3 = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
-    Key rowKey = new Key(metadataEntry);
-    
-    SortedMap<String,DataFileValue> origDatafileSizes = new TreeMap<String,DataFileValue>();
-    SortedMap<String,DataFileValue> highDatafileSizes = new TreeMap<String,DataFileValue>();
-    SortedMap<String,DataFileValue> lowDatafileSizes = new TreeMap<String,DataFileValue>();
-    scanner3.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
-    scanner3.setRange(new Range(rowKey, rowKey.followingKey(PartialKey.ROW)));
-    
-    for (Entry<Key,Value> entry : scanner3) {
-      if (entry.getKey().compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY) == 0) {
-        origDatafileSizes.put(entry.getKey().getColumnQualifier().toString(), new DataFileValue(entry.getValue().get()));
-      }
-    }
-    
-    splitDatafiles(table, metadataPrevEndRow, splitRatio, new HashMap<String,FileUtil.FileInfo>(), origDatafileSizes, lowDatafileSizes, highDatafileSizes,
-        highDatafilesToRemove);
-    
     // check to see if prev tablet exist in metadata tablet
     Key prevRowKey = new Key(new Text(KeyExtent.getMetadataEntry(table, metadataPrevEndRow)));
-    
+
     ScannerImpl scanner2 = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
     scanner2.setRange(new Range(prevRowKey, prevRowKey.followingKey(PartialKey.ROW)));
     
     if (!scanner2.iterator().hasNext()) {
-      log.debug("Prev tablet " + prevRowKey + " does not exist, need to create it " + metadataPrevEndRow + " " + prevPrevEndRow + " " + splitRatio);
-      Map<String,Long> bulkFiles = getBulkFilesLoaded(credentials, metadataEntry);
-      MetadataTable.addNewTablet(low, lowDirectory, tserver, lowDatafileSizes, bulkFiles, credentials, time, initFlushID, initCompactID, lock);
+      log.info("Rolling back incomplete split " + metadataEntry + " " + metadataPrevEndRow);
+      rollBackSplit(metadataEntry, KeyExtent.decodePrevEndRow(oper), credentials, lock);
+      return new KeyExtent(metadataEntry, KeyExtent.decodePrevEndRow(oper));
     } else {
-      log.debug("Prev tablet " + prevRowKey + " exist, do not need to add it");
-    }
-    
-    MetadataTable.finishSplit(metadataEntry, highDatafileSizes, highDatafilesToRemove, credentials, lock);
+      log.info("Finishing incomplete split " + metadataEntry + " " + metadataPrevEndRow);
+
+      List<String> highDatafilesToRemove = new ArrayList<String>();
+
+      Scanner scanner3 = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+      Key rowKey = new Key(metadataEntry);
+      
+      SortedMap<String,DataFileValue> origDatafileSizes = new TreeMap<String,DataFileValue>();
+      SortedMap<String,DataFileValue> highDatafileSizes = new TreeMap<String,DataFileValue>();
+      SortedMap<String,DataFileValue> lowDatafileSizes = new TreeMap<String,DataFileValue>();
+      scanner3.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+      scanner3.setRange(new Range(rowKey, rowKey.followingKey(PartialKey.ROW)));
+      
+      for (Entry<Key,Value> entry : scanner3) {
+        if (entry.getKey().compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY) == 0) {
+          origDatafileSizes.put(entry.getKey().getColumnQualifier().toString(), new DataFileValue(entry.getValue().get()));
+        }
+      }
+      
+      splitDatafiles(table, metadataPrevEndRow, splitRatio, new HashMap<String,FileUtil.FileInfo>(), origDatafileSizes, lowDatafileSizes, highDatafileSizes,
+          highDatafilesToRemove);
     
-    return low;
+      MetadataTable.finishSplit(metadataEntry, highDatafileSizes, highDatafilesToRemove, credentials, lock);
+      
+      return new KeyExtent(metadataEntry, KeyExtent.encodePrevEndRow(metadataPrevEndRow));
+    }
+
+
   }
   
   public static void splitDatafiles(Text table, Text midRow, double splitRatio, Map<String,FileUtil.FileInfo> firstAndLastRows,
@@ -597,13 +590,12 @@ public class MetadataTable extends org.a
   
   public static KeyExtent fixSplit(Text metadataEntry, SortedMap<ColumnFQ,Value> columns, TServerInstance tserver, TCredentials credentials, ZooLock lock)
       throws AccumuloException {
-    log.warn("Incomplete split " + metadataEntry + " attempting to fix");
+    log.info("Incomplete split " + metadataEntry + " attempting to fix");
     
     Value oper = columns.get(Constants.METADATA_OLD_PREV_ROW_COLUMN);
     
     if (columns.get(Constants.METADATA_SPLIT_RATIO_COLUMN) == null) {
-      log.warn("Metadata entry does not have split ratio (" + metadataEntry + ")");
-      return null;
+      throw new IllegalArgumentException("Metadata entry does not have split ratio (" + metadataEntry + ")");
     }
     
     double splitRatio = Double.parseDouble(new String(columns.get(Constants.METADATA_SPLIT_RATIO_COLUMN).get()));
@@ -611,15 +603,13 @@ public class MetadataTable extends org.a
     Value prevEndRowIBW = columns.get(Constants.METADATA_PREV_ROW_COLUMN);
     
     if (prevEndRowIBW == null) {
-      log.warn("Metadata entry does not have prev row (" + metadataEntry + ")");
-      return null;
+      throw new IllegalArgumentException("Metadata entry does not have prev row (" + metadataEntry + ")");
     }
     
     Value time = columns.get(Constants.METADATA_TIME_COLUMN);
     
     if (time == null) {
-      log.warn("Metadata entry does not have time (" + metadataEntry + ")");
-      return null;
+      throw new IllegalArgumentException("Metadata entry does not have time (" + metadataEntry + ")");
     }
     
     Value flushID = columns.get(Constants.METADATA_FLUSH_COLUMN);

Added: accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/tabletserver/CheckTabletMetadataTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/tabletserver/CheckTabletMetadataTest.java?rev=1466211&view=auto
==============================================================================
--- accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/tabletserver/CheckTabletMetadataTest.java (added)
+++ accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/tabletserver/CheckTabletMetadataTest.java Tue Apr  9 20:21:57 2013
@@ -0,0 +1,106 @@
+package org.apache.accumulo.server.tabletserver;
+
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class CheckTabletMetadataTest {
+  
+  private static Key nk(String row, ColumnFQ cfq) {
+    return new Key(new Text(row), cfq.getColumnFamily(), cfq.getColumnQualifier());
+  }
+  
+  private static Key nk(String row, Text cf, String cq) {
+    return new Key(row, cf.toString(), cq);
+  }
+
+  private static void put(TreeMap<Key,Value> tabletMeta, String row, ColumnFQ cfq, byte[] val) {
+    Key k = new Key(new Text(row), cfq.getColumnFamily(), cfq.getColumnQualifier());
+    tabletMeta.put(k, new Value(val));
+  }
+  
+  private static void put(TreeMap<Key,Value> tabletMeta, String row, Text cf, String cq, String val) {
+    Key k = new Key(new Text(row), cf, new Text(cq));
+    tabletMeta.put(k, new Value(val.getBytes()));
+  }
+
+  private static void assertFail(TreeMap<Key,Value> tabletMeta, KeyExtent ke, TServerInstance tsi) {
+    try {
+      Assert.assertNull(TabletServer.checkTabletMetadata(ke, tsi, tabletMeta, ke.getMetadataEntry()));
+    } catch (Exception e) {
+
+    }
+  }
+
+  private static void assertFail(TreeMap<Key,Value> tabletMeta, KeyExtent ke, TServerInstance tsi, Key keyToDelete) {
+    TreeMap<Key,Value> copy = new TreeMap<Key,Value>(tabletMeta);
+    Assert.assertNotNull(copy.remove(keyToDelete));
+    try {
+      Assert.assertNull(TabletServer.checkTabletMetadata(ke, tsi, copy, ke.getMetadataEntry()));
+    } catch (Exception e) {
+
+    }
+  }
+
+  @Test
+  public void testBadTabletMetadata() throws Exception {
+    
+    KeyExtent ke = new KeyExtent(new Text("1"), null, null);
+    
+    TreeMap<Key,Value> tabletMeta = new TreeMap<Key,Value>();
+    
+    put(tabletMeta, "1<", Constants.METADATA_PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(null).get());
+    put(tabletMeta, "1<", Constants.METADATA_DIRECTORY_COLUMN, "/t1".getBytes());
+    put(tabletMeta, "1<", Constants.METADATA_TIME_COLUMN, "M0".getBytes());
+    put(tabletMeta, "1<", Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY, "4", "127.0.0.1:9997");
+    
+    TServerInstance tsi = new TServerInstance("127.0.0.1:9997", 4);
+    
+    Assert.assertNotNull(TabletServer.checkTabletMetadata(ke, tsi, tabletMeta, ke.getMetadataEntry()));
+    
+    assertFail(tabletMeta, ke, new TServerInstance("127.0.0.1:9998", 4));
+    assertFail(tabletMeta, ke, new TServerInstance("127.0.0.1:9998", 5));
+    assertFail(tabletMeta, ke, new TServerInstance("127.0.0.1:9997", 5));
+    assertFail(tabletMeta, ke, new TServerInstance("127.0.0.2:9997", 4));
+    assertFail(tabletMeta, ke, new TServerInstance("127.0.0.2:9997", 5));
+    
+    assertFail(tabletMeta, new KeyExtent(new Text("1"), null, new Text("m")), tsi);
+    
+    assertFail(tabletMeta, new KeyExtent(new Text("1"), new Text("r"), new Text("m")), tsi);
+    
+    assertFail(tabletMeta, ke, tsi, nk("1<", Constants.METADATA_PREV_ROW_COLUMN));
+
+    assertFail(tabletMeta, ke, tsi, nk("1<", Constants.METADATA_DIRECTORY_COLUMN));
+    
+    assertFail(tabletMeta, ke, tsi, nk("1<", Constants.METADATA_TIME_COLUMN));
+    
+    assertFail(tabletMeta, ke, tsi, nk("1<", Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY, "4"));
+    
+    TreeMap<Key,Value> copy = new TreeMap<Key,Value>(tabletMeta);
+    put(copy, "1<", Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, "4", "127.0.0.1:9997");
+    assertFail(copy, ke, tsi);
+    assertFail(copy, ke, tsi, nk("1<", Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY, "4"));
+    
+    copy = new TreeMap<Key,Value>(tabletMeta);
+    put(copy, "1<", Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, "5", "127.0.0.1:9998");
+    assertFail(copy, ke, tsi);
+    put(copy, "1<", Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, "6", "127.0.0.1:9999");
+    assertFail(copy, ke, tsi);
+    
+    copy = new TreeMap<Key,Value>(tabletMeta);
+    put(copy, "1<", Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY, "5", "127.0.0.1:9998");
+    assertFail(copy, ke, tsi);
+    
+    assertFail(new TreeMap<Key,Value>(), ke, tsi);
+
+  }
+}

Modified: accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java?rev=1466211&r1=1466210&r2=1466211&view=diff
==============================================================================
--- accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java (original)
+++ accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java Tue Apr  9 20:21:57 2013
@@ -28,6 +28,8 @@ import java.util.SortedMap;
 import java.util.TreeMap;
 
 import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
 import org.apache.accumulo.core.client.impl.Writer;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.KeyExtent;
@@ -176,34 +178,32 @@ public class SplitRecoveryTest extends F
     if (steps >= 2)
       MetadataTable.finishSplit(high, highDatafileSizes, highDatafilesToRemove, SecurityConstants.getSystemCredentials(), zl);
     
-    SortedMap<KeyExtent,Text> vtiRet = TabletServer.verifyTabletInformation(extent, instance, null, "127.0.0.1:0", zl);
     
-    if (vtiRet.size() != 2) {
-      throw new Exception("verifyTabletInformation did not return two tablets, " + vtiRet.size());
-    }
-    
-    if (!vtiRet.containsKey(high) || !vtiRet.containsKey(low)) {
-      throw new Exception("verifyTabletInformation did not return correct tablets, " + vtiRet.keySet());
-    }
-    
-    ensureTabletHasNoUnexpectedMetadataEntries(low, lowDatafileSizes);
-    ensureTabletHasNoUnexpectedMetadataEntries(high, highDatafileSizes);
+    TabletServer.verifyTabletInformation(high, instance, null, "127.0.0.1:0", zl);
+
+    if (steps >= 1) {
+      ensureTabletHasNoUnexpectedMetadataEntries(low, lowDatafileSizes);
+      ensureTabletHasNoUnexpectedMetadataEntries(high, highDatafileSizes);
     
-    Map<String,Long> lowBulkFiles = MetadataTable.getBulkFilesLoaded(SecurityConstants.getSystemCredentials(), low);
-    Map<String,Long> highBulkFiles = MetadataTable.getBulkFilesLoaded(SecurityConstants.getSystemCredentials(), high);
+      Map<String,Long> lowBulkFiles = MetadataTable.getBulkFilesLoaded(SecurityConstants.getSystemCredentials(), low);
+      Map<String,Long> highBulkFiles = MetadataTable.getBulkFilesLoaded(SecurityConstants.getSystemCredentials(), high);
     
-    if (!lowBulkFiles.equals(highBulkFiles)) {
-      throw new Exception(" " + lowBulkFiles + " != " + highBulkFiles + " " + low + " " + high);
-    }
+      if (!lowBulkFiles.equals(highBulkFiles)) {
+        throw new Exception(" " + lowBulkFiles + " != " + highBulkFiles + " " + low + " " + high);
+      }
     
-    if (lowBulkFiles.size() == 0) {
-      throw new Exception(" no bulk files " + low);
+      if (lowBulkFiles.size() == 0) {
+        throw new Exception(" no bulk files " + low);
+      }
+    } else {
+      ensureTabletHasNoUnexpectedMetadataEntries(extent, mapFiles);
     }
   }
   
   private void ensureTabletHasNoUnexpectedMetadataEntries(KeyExtent extent, SortedMap<String,DataFileValue> expectedMapFiles) throws Exception {
-    SortedMap<Key,Value> tkv = new TreeMap<Key,Value>();
-    MetadataTable.getTabletAndPrevTabletKeyValues(tkv, extent, null, SecurityConstants.getSystemCredentials());
+    Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), SecurityConstants.getSystemCredentials(), Constants.METADATA_TABLE_ID,
+        Constants.NO_AUTHS);
+    scanner.setRange(extent.toMetadataRange());
     
     HashSet<ColumnFQ> expectedColumns = new HashSet<ColumnFQ>();
     expectedColumns.add(Constants.METADATA_DIRECTORY_COLUMN);
@@ -218,12 +218,12 @@ public class SplitRecoveryTest extends F
     expectedColumnFamilies.add(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
     expectedColumnFamilies.add(Constants.METADATA_BULKFILE_COLUMN_FAMILY);
     
-    Iterator<Key> iter = tkv.keySet().iterator();
+    Iterator<Entry<Key,Value>> iter = scanner.iterator();
     while (iter.hasNext()) {
-      Key key = iter.next();
+      Key key = iter.next().getKey();
       
       if (!key.getRow().equals(extent.getMetadataEntry())) {
-        continue;
+        throw new Exception("Tablet " + extent + " contained unexpected " + Constants.METADATA_TABLE_NAME + " entry " + key);
       }
       
       if (expectedColumnFamilies.contains(key.getColumnFamily())) {

Modified: accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java?rev=1466211&r1=1466210&r2=1466211&view=diff
==============================================================================
--- accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java (original)
+++ accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java Tue Apr  9 20:21:57 2013
@@ -27,8 +27,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Random;
-import java.util.Set;
-import java.util.SortedMap;
 import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
@@ -36,7 +34,6 @@ import java.util.concurrent.CountDownLat
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
-import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.cli.ScannerOpts;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
@@ -67,12 +64,12 @@ import org.apache.accumulo.core.security
 import org.apache.accumulo.core.security.thrift.TCredentials;
 import org.apache.accumulo.core.util.AddressUtil;
 import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.core.util.MetadataTable;
 import org.apache.accumulo.core.util.Stat;
 import org.apache.accumulo.server.ServerConstants;
 import org.apache.accumulo.server.cli.ClientOnRequiredTable;
 import org.apache.accumulo.server.conf.ServerConfiguration;
 import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.util.MetadataTable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FileStatus;
@@ -378,16 +375,8 @@ public class CollectTabletStats {
   private static List<String> getTabletFiles(TCredentials token, Instance zki, String tableId, KeyExtent ke) {
     List<String> files = new ArrayList<String>();
     
-    SortedMap<Key,Value> tkv = new TreeMap<Key,Value>();
-    MetadataTable.getTabletAndPrevTabletKeyValues(zki, tkv, ke, null, token);
-    
-    Set<Entry<Key,Value>> es = tkv.entrySet();
-    for (Entry<Key,Value> entry : es) {
-      if (entry.getKey().compareRow(ke.getMetadataEntry()) == 0) {
-        if (entry.getKey().compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY) == 0) {
-          files.add(ServerConstants.getTablesDir() + "/" + tableId + entry.getKey().getColumnQualifier());
-        }
-      }
+    for (String cq : MetadataTable.getDataFileSizes(ke, token).keySet()) {
+      files.add(ServerConstants.getTablesDir() + "/" + tableId + cq);
     }
     return files;
   }

Copied: accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/TestAccumuloSplitRecovery.java (from r1466018, accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/TestAccumulo1235.java)
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/TestAccumuloSplitRecovery.java?p2=accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/TestAccumuloSplitRecovery.java&p1=accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/TestAccumulo1235.java&r1=1466018&r2=1466211&rev=1466211&view=diff
==============================================================================
--- accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/TestAccumulo1235.java (original)
+++ accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/TestAccumuloSplitRecovery.java Tue Apr  9 20:21:57 2013
@@ -41,7 +41,7 @@ import org.junit.Before;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
-public class TestAccumulo1235 {
+public class TestAccumuloSplitRecovery {
   
   private static final String TABLE = "simple";
   public static TemporaryFolder folder = new TemporaryFolder();
@@ -81,44 +81,71 @@ public class TestAccumulo1235 {
   @Test
   public void test() throws Exception {
     
-    ZooKeeperInstance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
-    Connector connector = instance.getConnector("root", new PasswordToken(secret));
-    // create a table and put some data in it
-    connector.tableOperations().create(TABLE);
-    BatchWriter bw = connector.createBatchWriter(TABLE, new BatchWriterConfig());
-    bw.addMutation(m("a"));
-    bw.addMutation(m("b"));
-    bw.addMutation(m("c"));
-    bw.close();
-    // take the table offline
-    connector.tableOperations().offline(TABLE);
-    while (!isOffline(TABLE, connector))
-      UtilWaitThread.sleep(200);
+    for (int tn = 0; tn < 2; tn++) {
     
-    // poke a partial split into the !METADATA table
-    connector.securityOperations().grantTablePermission("root", Constants.METADATA_TABLE_NAME, TablePermission.WRITE);
-    String tableId = connector.tableOperations().tableIdMap().get(TABLE);
-    
-    KeyExtent extent = new KeyExtent(new Text(tableId), null, new Text("b"));
-    Mutation m = extent.getPrevRowUpdateMutation();
-    
-    Constants.METADATA_SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(0.5).getBytes()));
-    Constants.METADATA_OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(null));
-    bw = connector.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
-    bw.addMutation(m);
-    bw.close();
-    // bring the table online
-    connector.tableOperations().online(TABLE);
+      ZooKeeperInstance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
+      Connector connector = instance.getConnector("root", new PasswordToken(secret));
+      // create a table and put some data in it
+      connector.tableOperations().create(TABLE);
+      BatchWriter bw = connector.createBatchWriter(TABLE, new BatchWriterConfig());
+      bw.addMutation(m("a"));
+      bw.addMutation(m("b"));
+      bw.addMutation(m("c"));
+      bw.close();
+      // take the table offline
+      connector.tableOperations().offline(TABLE);
+      while (!isOffline(TABLE, connector))
+        UtilWaitThread.sleep(200);
+      
+      // poke a partial split into the !METADATA table
+      connector.securityOperations().grantTablePermission("root", Constants.METADATA_TABLE_NAME, TablePermission.WRITE);
+      String tableId = connector.tableOperations().tableIdMap().get(TABLE);
+      
+      KeyExtent extent = new KeyExtent(new Text(tableId), null, new Text("b"));
+      Mutation m = extent.getPrevRowUpdateMutation();
+      
+      Constants.METADATA_SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(0.5).getBytes()));
+      Constants.METADATA_OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(null));
+      bw = connector.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
+      bw.addMutation(m);
+      
+      if (tn == 1) {
+        
+        bw.flush();
+        
+        Scanner scanner = connector.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+        scanner.setRange(extent.toMetadataRange());
+        scanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+        
+        KeyExtent extent2 = new KeyExtent(new Text(tableId), new Text("b"), null);
+        m = extent2.getPrevRowUpdateMutation();
+        Constants.METADATA_DIRECTORY_COLUMN.put(m, new Value("/t2".getBytes()));
+        Constants.METADATA_TIME_COLUMN.put(m, new Value("M0".getBytes()));
+        
+        for (Entry<Key,Value> entry : scanner) {
+          m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, entry.getKey().getColumnQualifier(), entry.getValue());
+        }
+        
+        bw.addMutation(m);
+      }
+      
+      bw.close();
+      // bring the table online
+      connector.tableOperations().online(TABLE);
+      
+      // verify the tablets went online
+      Scanner scanner = connector.createScanner(TABLE, Constants.NO_AUTHS);
+      int i = 0;
+      String expected[] = {"a", "b", "c"};
+      for (Entry<Key,Value> entry : scanner) {
+        assertEquals(expected[i], entry.getKey().getRow().toString());
+        i++;
+      }
+      assertEquals(3, i);
+
+      connector.tableOperations().delete(TABLE);
     
-    // verify the tablets went online
-    Scanner scanner = connector.createScanner(TABLE, Constants.NO_AUTHS);
-    int i = 0;
-    String expected[] = { "a", "b", "c" };
-    for (Entry<Key,Value> entry: scanner) {
-      assertEquals(expected[i], entry.getKey().getRow().toString());
-      i++;
     }
-    assertEquals(3, i);
   }
   
 }



Mime
View raw message