accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1465156 - in /accumulo/trunk: ./ assemble/ core/ core/src/main/java/org/apache/accumulo/core/client/impl/ core/src/main/java/org/apache/accumulo/core/util/ core/src/test/java/org/apache/accumulo/core/client/impl/ examples/ fate/src/main/ja...
Date Fri, 05 Apr 2013 23:17:48 GMT
Author: kturner
Date: Fri Apr  5 23:17:47 2013
New Revision: 1465156

URL: http://svn.apache.org/r1465156
Log:
ACCUMULO-1248 made tablet location cache throw an exception if a tablet has multiple locations

Modified:
    accumulo/trunk/   (props changed)
    accumulo/trunk/assemble/   (props changed)
    accumulo/trunk/core/   (props changed)
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java
    accumulo/trunk/examples/   (props changed)
    accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java   (props changed)
    accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java 
 (props changed)
    accumulo/trunk/server/   (props changed)
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
    accumulo/trunk/src/   (props changed)

Propchange: accumulo/trunk/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5:r1464818-1465064,1465066-1465107,1465109-1465155

Propchange: accumulo/trunk/assemble/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/assemble:r1464818-1465064,1465066-1465107,1465109-1465155

Propchange: accumulo/trunk/core/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/core:r1464818-1465064,1465066-1465107,1465109-1465155

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java?rev=1465156&r1=1465155&r2=1465156&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
Fri Apr  5 23:17:47 2013
@@ -18,6 +18,7 @@ package org.apache.accumulo.core.client.
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -41,6 +42,8 @@ import org.apache.accumulo.core.data.Key
 import org.apache.accumulo.core.data.PartialKey;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
 import org.apache.accumulo.core.security.thrift.TCredentials;
 import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
 import org.apache.accumulo.core.util.MetadataTable;
@@ -81,16 +84,27 @@ public class MetadataLocationObtainer im
       
       Range range = new Range(row, true, stopRow, true);
       
+      TreeMap<Key,Value> encodedResults = new TreeMap<Key,Value>();
       TreeMap<Key,Value> results = new TreeMap<Key,Value>();
       
-      // System.out.println(range);
+      // Use the whole row iterator so that a partial mutations is not read. The code that
extracts locations for tablets does a sanity check to ensure there is
+      // only one location. Reading a partial mutation could make it appear there are multiple
locations when there are not.
+      List<IterInfo> serverSideIteratorList = new ArrayList<IterInfo>();
+      serverSideIteratorList.add(new IterInfo(10000, WholeRowIterator.class.getName(), "WRI"));
+      Map<String,Map<String,String>> serverSideIteratorOptions = Collections.emptyMap();
       
-      boolean more = ThriftScanner.getBatchFromServer(credentials, range, src.tablet_extent,
src.tablet_location, results, locCols, Constants.SCAN_BATCH_SIZE,
-          Constants.NO_AUTHS, false, instance.getConfiguration());
+      boolean more = ThriftScanner.getBatchFromServer(credentials, range, src.tablet_extent,
src.tablet_location, encodedResults, locCols,
+          serverSideIteratorList, serverSideIteratorOptions, Constants.SCAN_BATCH_SIZE, Constants.NO_AUTHS,
false, instance.getConfiguration());
+
+      decodeRows(encodedResults, results);
+
       if (more && results.size() == 1) {
         range = new Range(results.lastKey().followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME),
true, new Key(stopRow).followingKey(PartialKey.ROW), false);
-        more = ThriftScanner.getBatchFromServer(credentials, range, src.tablet_extent, src.tablet_location,
results, locCols, Constants.SCAN_BATCH_SIZE,
-            Constants.NO_AUTHS, false, instance.getConfiguration());
+        encodedResults.clear();
+        more = ThriftScanner.getBatchFromServer(credentials, range, src.tablet_extent, src.tablet_location,
encodedResults, locCols, serverSideIteratorList,
+            serverSideIteratorOptions, Constants.SCAN_BATCH_SIZE, Constants.NO_AUTHS, false,
instance.getConfiguration());
+        
+        decodeRows(encodedResults, results);
       }
       
       if (opTimer != null)
@@ -122,10 +136,20 @@ public class MetadataLocationObtainer im
     
     return null;
   }
+
+  private void decodeRows(TreeMap<Key,Value> encodedResults, TreeMap<Key,Value>
results) throws AccumuloException {
+    for (Entry<Key,Value> entry : encodedResults.entrySet()) {
+      try {
+        results.putAll(WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()));
+      } catch (IOException e) {
+        throw new AccumuloException(e);
+      }
+    }
+  }
   
   @Override
-  public List<TabletLocation> lookupTablets(String tserver, Map<KeyExtent,List<Range>>
tabletsRanges, TabletLocator parent, TCredentials credentials) throws AccumuloSecurityException,
-      AccumuloException {
+  public List<TabletLocation> lookupTablets(String tserver, Map<KeyExtent,List<Range>>
tabletsRanges, TabletLocator parent, TCredentials credentials)
+      throws AccumuloSecurityException, AccumuloException {
     
     final TreeMap<Key,Value> results = new TreeMap<Key,Value>();
     
@@ -136,13 +160,20 @@ public class MetadataLocationObtainer im
       @Override
       public void receive(List<Entry<Key,Value>> entries) {
         for (Entry<Key,Value> entry : entries) {
-          results.put(entry.getKey(), entry.getValue());
+          try {
+            results.putAll(WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()));
+          } catch (IOException e) {
+            throw new RuntimeException(e);
+          }
         }
       }
     };
     
     ScannerOptions opts = new ScannerOptions();
     opts.fetchedColumns = locCols;
+    opts.serverSideIteratorList = new ArrayList<IterInfo>();
+    opts.serverSideIteratorList.add(new IterInfo(10000, WholeRowIterator.class.getName(),
"WRI")); // see comment in lookupTablet about why iterator is
+                                                                                        
          // used
     
     Map<KeyExtent,List<Range>> unscanned = new HashMap<KeyExtent,List<Range>>();
     Map<KeyExtent,List<Range>> failures = new HashMap<KeyExtent,List<Range>>();
@@ -162,7 +193,7 @@ public class MetadataLocationObtainer im
       log.trace("lookupTablets failed server=" + tserver, e);
       throw e;
     }
-    
+
     SortedMap<KeyExtent,Text> metadata = MetadataTable.getMetadataLocationEntries(results).getFirst();
     
     for (Entry<KeyExtent,Text> entry : metadata.entrySet()) {

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java?rev=1465156&r1=1465155&r2=1465156&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
Fri Apr  5 23:17:47 2013
@@ -55,7 +55,6 @@ import org.apache.accumulo.core.tabletse
 import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
 import org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException;
 import org.apache.accumulo.core.util.OpTimer;
-import org.apache.accumulo.core.util.TextUtil;
 import org.apache.accumulo.core.util.ThriftUtil;
 import org.apache.accumulo.core.util.UtilWaitThread;
 import org.apache.accumulo.trace.instrument.Span;
@@ -71,7 +70,6 @@ import org.apache.thrift.TServiceClient;
 
 
 public class ThriftScanner {
-  private static final byte[] EMPTY_BYTES = new byte[0];
   private static final Logger log = Logger.getLogger(ThriftScanner.class);
   
   public static final Map<TabletType,Set<String>> serversWaitedForWrites = new
EnumMap<TabletType,Set<String>>(TabletType.class);
@@ -82,38 +80,9 @@ public class ThriftScanner {
     }
   }
   
-  public static boolean getBatchFromServer(TCredentials credentials, Text startRow, KeyExtent
extent, String server, SortedMap<Key,Value> results,
-      SortedSet<Column> fetchedColumns, boolean skipStartKey, int size, Authorizations
authorizations, boolean retry, AccumuloConfiguration conf)
-      throws AccumuloException, AccumuloSecurityException, NotServingTabletException {
-    Key startKey;
-    
-    if (fetchedColumns.size() > 0) {
-      byte[] cf = fetchedColumns.first().columnFamily;
-      byte[] cq = fetchedColumns.first().columnQualifier;
-      byte[] cv = fetchedColumns.first().columnVisibility;
-      
-      startKey = new Key(TextUtil.getBytes(startRow), cf, cq == null ? EMPTY_BYTES : cq,
cv == null ? EMPTY_BYTES : cv, Long.MAX_VALUE);
-      
-    } else
-      startKey = new Key(startRow);
-    
-    if (skipStartKey)
-      startKey = startKey.followingKey(PartialKey.ROW);
-    else
-      startKey.setTimestamp(Long.MAX_VALUE);
-    
-    return getBatchFromServer(credentials, startKey, (Key) null, extent, server, results,
fetchedColumns, size, authorizations, retry, conf);
-  }
-  
-  static boolean getBatchFromServer(TCredentials credentials, Key key, Key endKey, KeyExtent
extent, String server, SortedMap<Key,Value> results,
-      SortedSet<Column> fetchedColumns, int size, Authorizations authorizations, boolean
retry, AccumuloConfiguration conf) throws AccumuloException,
-      AccumuloSecurityException, NotServingTabletException {
-    return getBatchFromServer(credentials, new Range(key, true, endKey, true), extent, server,
results, fetchedColumns, size, authorizations, retry, conf);
-  }
-  
   static boolean getBatchFromServer(TCredentials credentials, Range range, KeyExtent extent,
String server, SortedMap<Key,Value> results,
-      SortedSet<Column> fetchedColumns, int size, Authorizations authorizations, boolean
retry, AccumuloConfiguration conf) throws AccumuloException,
-      AccumuloSecurityException, NotServingTabletException {
+      SortedSet<Column> fetchedColumns, List<IterInfo> serverSideIteratorList,
Map<String,Map<String,String>> serverSideIteratorOptions, int size,
+      Authorizations authorizations, boolean retry, AccumuloConfiguration conf) throws AccumuloException,
AccumuloSecurityException, NotServingTabletException {
     if (server == null)
       throw new AccumuloException(new IOException());
     
@@ -121,10 +90,9 @@ public class ThriftScanner {
       TInfo tinfo = Tracer.traceInfo();
       TabletClientService.Client client = ThriftUtil.getTServerClient(server, conf);
       try {
-        List<IterInfo> emptyList = Collections.emptyList();
-        Map<String,Map<String,String>> emptyMap = Collections.emptyMap();
         // not reading whole rows (or stopping on row boundries) so there is no need to enable
isolation below
-        ScanState scanState = new ScanState(credentials, extent.getTableId(), authorizations,
range, fetchedColumns, size, emptyList, emptyMap, false);
+        ScanState scanState = new ScanState(credentials, extent.getTableId(), authorizations,
range, fetchedColumns, size, serverSideIteratorList,
+            serverSideIteratorOptions, false);
         
         TabletType ttype = TabletType.type(extent);
         boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(server);

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java?rev=1465156&r1=1465155&r2=1465156&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java Fri
Apr  5 23:17:47 2013
@@ -150,10 +150,14 @@ public class MetadataTable {
       colq = key.getColumnQualifier(colq);
       
       // interpret the row id as a key extent
-      if (colf.equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY) || colf.equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY))
+      if (colf.equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY) || colf.equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY))
{
+        if (location != null) {
+          throw new IllegalStateException("Tablet has multiple locations : " + lastRowFromKey);
+        }
         location = new Text(val.toString());
-      else if (Constants.METADATA_PREV_ROW_COLUMN.equals(colf, colq))
+      } else if (Constants.METADATA_PREV_ROW_COLUMN.equals(colf, colq)) {
         prevRow = new Value(val);
+      }
       
       if (prevRow != null) {
         ke = new KeyExtent(key.getRow(), prevRow);

Modified: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java?rev=1465156&r1=1465155&r2=1465156&view=diff
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java
(original)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java
Fri Apr  5 23:17:47 2013
@@ -605,7 +605,7 @@ public class TabletLocatorImplTest exten
     }
   }
   
-  static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyExtent ke,
String location) {
+  static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyExtent ke,
String location, String instance) {
     Map<KeyExtent,SortedMap<Key,Value>> tablets = tservers.tservers.get(server);
     if (tablets == null) {
       tablets = new HashMap<KeyExtent,SortedMap<Key,Value>>();
@@ -622,15 +622,20 @@ public class TabletLocatorImplTest exten
     Value per = KeyExtent.encodePrevEndRow(ke.getPrevEndRow());
     
     if (location != null) {
-      Key lk = new Key(mr, Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, new Text());
+      if (instance == null)
+        instance = "";
+      Key lk = new Key(mr, Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, new Text(instance));
       tabletData.put(lk, new Value(location.getBytes()));
     }
     
     Key pk = new Key(mr, Constants.METADATA_PREV_ROW_COLUMN.getColumnFamily(), Constants.METADATA_PREV_ROW_COLUMN.getColumnQualifier());
     tabletData.put(pk, per);
-    
   }
   
+  static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyExtent ke,
String location) {
+    setLocation(tservers, server, tablet, ke, location, "");
+  }
+
   static void deleteServer(TServers tservers, String server) {
     tservers.tservers.remove(server);
     
@@ -1269,4 +1274,25 @@ public class TabletLocatorImplTest exten
     locateTabletTest(tab0TabletCache, "a", ke1, "tserver7", credential);
     
   }
+  
+  public void testAccumulo1248() throws Exception {
+    TServers tservers = new TServers();
+    TabletLocatorImpl metaCache = createLocators(tservers, "tserver1", "tserver2", "foo");
+    
+    KeyExtent ke1 = nke("foo", null, null);
+    
+    // set two locations for a tablet, this is not supposed to happen. The metadata cache
should throw an exception if it sees this rather than caching one of
+    // the locations.
+    setLocation(tservers, "tserver2", MTE, ke1, "L1", "I1");
+    setLocation(tservers, "tserver2", MTE, ke1, "L2", "I2");
+    
+    try {
+      metaCache.locateTablet(new Text("a"), false, false, credential);
+      assertTrue(false);
+    } catch (Exception e) {
+      
+    }
+
+
+  }
 }

Propchange: accumulo/trunk/examples/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/examples:r1464818-1465064,1465066-1465107,1465109-1465155

Propchange: accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java:r1464818-1465064,1465066-1465107,1465109-1465155

Propchange: accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java:r1464818-1465064,1465066-1465107,1465109-1465155

Propchange: accumulo/trunk/server/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/server:r1464818-1465064,1465066-1465107,1465109-1465155

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java?rev=1465156&r1=1465155&r2=1465156&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
Fri Apr  5 23:17:47 2013
@@ -30,7 +30,6 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.SortedMap;
-import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 
@@ -47,10 +46,7 @@ import org.apache.accumulo.core.client.S
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.client.impl.BatchWriterImpl;
 import org.apache.accumulo.core.client.impl.ScannerImpl;
-import org.apache.accumulo.core.client.impl.ThriftScanner;
 import org.apache.accumulo.core.client.impl.Writer;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.data.Column;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.KeyExtent;
 import org.apache.accumulo.core.data.Mutation;
@@ -61,7 +57,6 @@ import org.apache.accumulo.core.file.Fil
 import org.apache.accumulo.core.security.CredentialHelper;
 import org.apache.accumulo.core.security.thrift.TCredentials;
 import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
-import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
 import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.core.util.ColumnFQ;
 import org.apache.accumulo.core.util.FastFormat;
@@ -324,21 +319,6 @@ public class MetadataTable extends org.a
     return results;
   }
   
-  public static boolean getBatchFromRootTablet(AccumuloConfiguration conf, TCredentials credentials,
Text startRow, SortedMap<Key,Value> results,
-      SortedSet<Column> columns, boolean skipStartRow, int size) throws AccumuloSecurityException
{
-    while (true) {
-      try {
-        return ThriftScanner.getBatchFromServer(credentials, startRow, Constants.ROOT_TABLET_EXTENT,
HdfsZooInstance.getInstance().getRootTabletLocation(),
-            results, columns, skipStartRow, size, Constants.NO_AUTHS, true, conf);
-      } catch (NotServingTabletException e) {
-        UtilWaitThread.sleep(100);
-      } catch (AccumuloException e) {
-        UtilWaitThread.sleep(100);
-      }
-    }
-    
-  }
-  
   public static boolean recordRootTabletLocation(String address) {
     IZooReaderWriter zoo = ZooReaderWriter.getInstance();
     for (int i = 0; i < SAVE_ROOT_TABLET_RETRIES; i++) {

Propchange: accumulo/trunk/src/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/src:r1464818-1465064,1465066-1465107,1465109-1465155



Mime
View raw message