accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [2/5] accumulo git commit: ACCUMULO-3314 Add a fair volume chooser to stabilize RewriteTabletDirectoriesIT
Date Fri, 07 Nov 2014 18:31:08 GMT
ACCUMULO-3314 Add a fair volume chooser to stabilize RewriteTabletDirectoriesIT

The RandomVolumeChooser creates failure potential for the test
due to its random nature. A volume chooser that is stable should
result in a stable test.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/62e3e9b5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/62e3e9b5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/62e3e9b5

Branch: refs/heads/master
Commit: 62e3e9b53b35da98c9fff32a1366911c94dd8c7b
Parents: fa978b0
Author: Josh Elser <elserj@apache.org>
Authored: Fri Nov 7 13:13:30 2014 -0500
Committer: Josh Elser <elserj@apache.org>
Committed: Fri Nov 7 13:13:30 2014 -0500

----------------------------------------------------------------------
 .../apache/accumulo/test/FairVolumeChooser.java | 48 ++++++++++++++++++++
 .../test/RewriteTabletDirectoriesIT.java        | 19 ++++++--
 2 files changed, 63 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/62e3e9b5/test/src/main/java/org/apache/accumulo/test/FairVolumeChooser.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/FairVolumeChooser.java b/test/src/main/java/org/apache/accumulo/test/FairVolumeChooser.java
new file mode 100644
index 0000000..9eb0c84
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/FairVolumeChooser.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.accumulo.server.fs.VolumeChooser;
+
+/**
+ * Try to assign some fairness to choosing Volumes. Intended for tests, not for production
+ */
+public class FairVolumeChooser implements VolumeChooser {
+
+  private final ConcurrentHashMap<Integer,Integer> optionLengthToLastChoice = new ConcurrentHashMap<Integer,Integer>();
+
+  @Override
+  public String choose(String[] options) {
+    int currentChoice;
+    Integer lastChoice = optionLengthToLastChoice.get(options.length);
+    if (null == lastChoice) {
+      currentChoice = 0;
+    } else {
+      currentChoice = lastChoice + 1;
+      if (currentChoice >= options.length) {
+        currentChoice = 0;
+      }
+    }
+
+    optionLengthToLastChoice.put(options.length, currentChoice);
+
+    return options[currentChoice];
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/62e3e9b5/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java b/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
index fe233ee..cacee7a 100644
--- a/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.test;
 
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.BufferedOutputStream;
@@ -52,7 +53,7 @@ import org.junit.Test;
 
 // ACCUMULO-3263
 public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
-  
+
   @Override
   public int defaultTimeoutSeconds() {
     return 4 * 60;
@@ -71,6 +72,8 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
     v1 = new Path("file://" + v1f.getAbsolutePath());
     v2 = new Path("file://" + v2f.getAbsolutePath());
 
+    // Use a VolumeChooser which should be more fair
+    cfg.setProperty(Property.GENERAL_VOLUME_CHOOSER, FairVolumeChooser.class.getName());
     // Run MAC on two locations in the local file system
     cfg.setProperty(Property.INSTANCE_VOLUMES, v1.toString());
     hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
@@ -83,6 +86,8 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
     c.securityOperations().grantTablePermission(c.whoami(), MetadataTable.NAME, TablePermission.WRITE);
     final String tableName = getUniqueNames(1)[0];
     c.tableOperations().create(tableName);
+
+    // Write some data to a table and add some splits
     BatchWriter bw = c.createBatchWriter(tableName, null);
     final SortedSet<Text> splits = new TreeSet<Text>();
     for (String split : "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(","))
{
@@ -97,12 +102,13 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
     BatchScanner scanner = c.createBatchScanner(MetadataTable.NAME, Authorizations.EMPTY,
1);
     DIRECTORY_COLUMN.fetch(scanner);
     String tableId = c.tableOperations().tableIdMap().get(tableName);
+    assertNotNull("TableID for " + tableName + " was null", tableId);
     scanner.setRanges(Collections.singletonList(TabletsSection.getRange(tableId)));
     // verify the directory entries are all on v1, make a few entries relative
     bw = c.createBatchWriter(MetadataTable.NAME, null);
     int count = 0;
     for (Entry<Key,Value> entry : scanner) {
-      assertTrue(entry.getValue().toString().contains(v1.toString()));
+      assertTrue("Expected " + entry.getValue() + " to contain " + v1, entry.getValue().toString().contains(v1.toString()));
       count++;
       if (count % 2 == 0) {
         String parts[] = entry.getValue().toString().split("/");
@@ -146,12 +152,17 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT {
         v2Count++;
       }
     }
+
+    log.info("Count for volume1: " + v1Count);
+    log.info("Count for volume2: " + v2Count);
+
     assertEquals(splits.size() + 1, v1Count + v2Count);
-    assertTrue(Math.abs(v1Count - v2Count) < 10);
+    // a fair chooser will differ by less than count(volumes)
+    assertTrue("Expected the number of files to differ between volumes by less than 10. "
+ v1Count + " " + v2Count, Math.abs(v1Count - v2Count) < 2);
     // verify we can read the old data
     count = 0;
     for (Entry<Key,Value> entry : c.createScanner(tableName, Authorizations.EMPTY))
{
-      assertTrue(splits.contains(entry.getKey().getRow()));
+      assertTrue("Found unexpected entry in table: " + entry, splits.contains(entry.getKey().getRow()));
       count++;
     }
     assertEquals(splits.size(), count);


Mime
View raw message