accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1467010 - in /accumulo/branches/1.5: server/src/main/java/org/apache/accumulo/server/master/ server/src/main/java/org/apache/accumulo/server/master/state/ test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ test/src/test/jav...
Date Thu, 11 Apr 2013 17:56:58 GMT
Author: ecn
Date: Thu Apr 11 17:56:57 2013
New Revision: 1467010

URL: http://svn.apache.org/r1467010
Log:
ACCUMULO-1264 tweak the scan range when looking for !METADATA tablet consistency check; add
!METADATA table merge/split to the randomwalk test

Added:
    accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java 
 (with props)
Modified:
    accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java
    accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
    accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java
    accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java

Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1467010&r1=1467009&r2=1467010&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java
(original)
+++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java
Thu Apr 11 17:56:57 2013
@@ -1203,6 +1203,7 @@ public class Master implements LiveTServ
       }
       // Handle merge transitions
       if (mergeInfo.getRange() != null) {
+        log.debug("mergeInfo overlaps: " + extent + " " + mergeInfo.overlaps(extent));
         if (mergeInfo.overlaps(extent)) {
           switch (mergeInfo.getState()) {
             case NONE:
@@ -2094,8 +2095,11 @@ public class Master implements LiveTServ
     });
     
     TCredentials systemAuths = SecurityConstants.getSystemCredentials();
-    final TabletStateStore stores[] = {new ZooTabletStateStore(new ZooStore(zroot)), new
RootTabletStateStore(instance, systemAuths, this),
-        new MetaDataStateStore(instance, systemAuths, this)};
+    final TabletStateStore stores[] = {
+        new ZooTabletStateStore(new ZooStore(zroot)), 
+        new RootTabletStateStore(instance, systemAuths, this),
+        new MetaDataStateStore(instance, systemAuths, this)
+    };
     watchers.add(new TabletGroupWatcher(stores[2], null));
     watchers.add(new TabletGroupWatcher(stores[1], watchers.get(0)));
     watchers.add(new TabletGroupWatcher(stores[0], watchers.get(1)));

Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java?rev=1467010&r1=1467009&r2=1467010&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
(original)
+++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
Thu Apr 11 17:56:57 2013
@@ -28,6 +28,7 @@ import org.apache.accumulo.core.client.S
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
@@ -182,11 +183,14 @@ public class MergeStats {
     Text tableId = extent.getTableId();
     Text first = KeyExtent.getMetadataEntry(tableId, start);
     Range range = new Range(first, false, null, true);
-    if (extent.isMeta())
-      range = new Range(Constants.METADATA_ROOT_TABLET_KEYSPACE);
+    if (extent.isMeta()) {
+      // don't go off the root tablet
+      range = new Range(new Key(first).followingKey(PartialKey.ROW), false, Constants.METADATA_ROOT_TABLET_KEYSPACE.getEndKey(),
false);
+    }
     scanner.setRange(range);
     KeyExtent prevExtent = null;
 
+    log.debug("Scanning range " + range);
     for (Entry<Key,Value> entry : scanner) {
       TabletLocationState tls;
       try {
@@ -195,24 +199,30 @@ public class MergeStats {
         log.error(e, e);
         return false;
       }
+      log.debug("consistency check: " + tls + " walogs " + tls.walogs.size());
       if (!tls.extent.getTableId().equals(tableId)) {
         break;
       }
 
       if (!tls.walogs.isEmpty() && verify.getMergeInfo().needsToBeChopped(tls.extent))
{
+        log.debug("failing consistency: needs to be chopped" + tls.extent);
         return false;
       }
 
       if (prevExtent == null) {
         // this is the first tablet observed, it must be offline and its prev row must be
less than the start of the merge range
         if (tls.extent.getPrevEndRow() != null && tls.extent.getPrevEndRow().compareTo(start)
> 0) {
+          log.debug("failing consistency: prev row is too high " + start);
           return false;
         }
         
-        if (tls.getState(master.onlineTabletServers()) != TabletState.UNASSIGNED)
+        if (tls.getState(master.onlineTabletServers()) != TabletState.UNASSIGNED) {
+          log.debug("failing consistency: assigned or hosted " + tls);
           return false;
+        }
         
       } else if (!tls.extent.isPreviousExtent(prevExtent)) {
+        log.debug("hole in !METADATA");
         return false;
       }
       
@@ -224,6 +234,9 @@ public class MergeStats {
         break;
       }
     }
+    log.debug("chopped " + chopped + " v.chopped " + verify.chopped + 
+        " unassigned " + unassigned + " v.unassigned " + verify.unassigned +
+        " verify.total " + verify.total);
     return chopped == verify.chopped && unassigned == verify.unassigned &&
unassigned == verify.total;
   }
   

Modified: accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java?rev=1467010&r1=1467009&r2=1467010&view=diff
==============================================================================
--- accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java
(original)
+++ accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java
Thu Apr 11 17:56:57 2013
@@ -16,11 +16,13 @@
  */
 package org.apache.accumulo.test.randomwalk.concurrent;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 import java.util.Random;
 import java.util.TreeSet;
 
+import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.client.TableOfflineException;
@@ -38,7 +40,8 @@ public class AddSplits extends Test {
     
     @SuppressWarnings("unchecked")
     List<String> tableNames = (List<String>) state.get("tables");
-    
+    tableNames = new ArrayList<String>(tableNames);
+    tableNames.add(Constants.METADATA_TABLE_NAME);
     String tableName = tableNames.get(rand.nextInt(tableNames.size()));
     
     TreeSet<Text> splits = new TreeSet<Text>();

Modified: accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java?rev=1467010&r1=1467009&r2=1467010&view=diff
==============================================================================
--- accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java
(original)
+++ accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java
Thu Apr 11 17:56:57 2013
@@ -16,11 +16,13 @@
  */
 package org.apache.accumulo.test.randomwalk.concurrent;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 import java.util.Random;
 import java.util.TreeSet;
 
+import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.client.TableOfflineException;
@@ -38,7 +40,8 @@ public class Merge extends Test {
     
     @SuppressWarnings("unchecked")
     List<String> tableNames = (List<String>) state.get("tables");
-    
+    tableNames = new ArrayList<String>(tableNames);
+    tableNames.add(Constants.METADATA_TABLE_NAME);
     String tableName = tableNames.get(rand.nextInt(tableNames.size()));
     
     // TODO need to sometimes do null start and end ranges

Added: accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java?rev=1467010&view=auto
==============================================================================
--- accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java (added)
+++ accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java Thu
Apr 11 17:56:57 2013
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.hadoop.io.Text;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class MetaSplitTest {
+  
+  private static String secret = "superSecret";
+  public static TemporaryFolder folder = new TemporaryFolder();
+  public static MiniAccumuloCluster cluster;
+
+  
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    folder.create();
+    MiniAccumuloConfig cfg = new MiniAccumuloConfig(folder.newFolder("miniAccumulo"), secret);
+    cluster = new MiniAccumuloCluster(cfg);
+    cluster.start();
+
+  }
+  
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    cluster.stop();
+    folder.delete();
+  }
+  
+  private void addSplits(TableOperations opts, String ... points) throws Exception {
+    SortedSet<Text> splits = new TreeSet<Text>();
+    for (String point : points) {
+      splits.add(new Text(point));
+    }
+    opts.addSplits(Constants.METADATA_TABLE_NAME, splits);
+  }
+  
+  @Test(timeout = 60000)
+  public void testMetaSplit() throws Exception {
+    Instance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers());
+    Connector connector = instance.getConnector("root", new PasswordToken(secret));
+    TableOperations opts = connector.tableOperations();
+    for (int i = 1; i <= 10; i++) {
+      opts.create("" + i);
+    }
+    assertEquals(2, opts.listSplits(Constants.METADATA_TABLE_NAME).size());
+    addSplits(opts, "4 5 6 7 8".split(" "));
+    assertEquals(7, opts.listSplits(Constants.METADATA_TABLE_NAME).size());
+    opts.merge(Constants.METADATA_TABLE_NAME, new Text("6"), new Text("9"));
+    assertEquals(5, opts.listSplits(Constants.METADATA_TABLE_NAME).size());
+    addSplits(opts, "44 55 66 77 88".split(" "));
+    assertEquals(10, opts.listSplits(Constants.METADATA_TABLE_NAME).size());
+    opts.merge(Constants.METADATA_TABLE_NAME, new Text("5"), new Text("7"));
+    assertEquals(7, opts.listSplits(Constants.METADATA_TABLE_NAME).size());
+    opts.merge(Constants.METADATA_TABLE_NAME, null, null);
+    assertEquals(1, opts.listSplits(Constants.METADATA_TABLE_NAME).size());
+  }
+  
+}

Propchange: accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message