accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1393868 - in /accumulo/branches/1.4: src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java test/system/auto/simple/fateStartvation.py
Date Thu, 04 Oct 2012 01:27:16 GMT
Author: kturner
Date: Thu Oct  4 01:27:15 2012
New Revision: 1393868

URL: http://svn.apache.org/viewvc?rev=1393868&view=rev
Log:
ACCUMULO-779 fixed Fate starvation bug

Added:
    accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
    accumulo/branches/1.4/test/system/auto/simple/fateStartvation.py   (with props)
Modified:
    accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java

Modified: accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java?rev=1393868&r1=1393867&r2=1393868&view=diff
==============================================================================
--- accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java
(original)
+++ accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java
Thu Oct  4 01:27:15 2012
@@ -47,6 +47,7 @@ public class ZooStore<T> implements TSto
   
   private String path;
   private IZooReaderWriter zk;
+  private String lastReserved = "";
   private Set<Long> reserved;
   private Map<Long,Long> defered;
   private SecureRandom idgenerator;
@@ -123,20 +124,33 @@ public class ZooStore<T> implements TSto
           events = statusChangeEvents;
         }
         
-        List<String> txdirs = zk.getChildren(path);
+        List<String> txdirs = new ArrayList<String>(zk.getChildren(path));
+        Collections.sort(txdirs);
+        
+        synchronized (this) {
+          if (txdirs.size() > 0 && txdirs.get(txdirs.size() - 1).compareTo(lastReserved)
<= 0)
+            lastReserved = "";
+        }
         
         for (String txdir : txdirs) {
           long tid = parseTid(txdir);
           
           synchronized (this) {
+            // this check makes reserve pick up where it left off, so that it cycles through
all as it is repeatedly called.... failing to do so can lead to
+            // starvation where fate ops that sort higher and hold a lock are never reserved.
+            if (txdir.compareTo(lastReserved) <= 0)
+              continue;
+
             if (defered.containsKey(tid)) {
               if (defered.get(tid) < System.currentTimeMillis())
                 defered.remove(tid);
               else
                 continue;
             }
-            if (!reserved.contains(tid))
+            if (!reserved.contains(tid)) {
               reserved.add(tid);
+              lastReserved = txdir;
+            }
             else
               continue;
           }

Added: accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java?rev=1393868&view=auto
==============================================================================
--- accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
(added)
+++ accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
Thu Oct  4 01:27:15 2012
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.test.functional;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.accumulo.server.test.TestIngest;
+import org.apache.hadoop.io.Text;
+
+/**
+ * See ACCUMULO-779
+ */
+public class FateStarvationTest extends FunctionalTest {
+  
+  @Override
+  public void cleanup() throws Exception {}
+  
+  @Override
+  public Map<String,String> getInitialConfig() {
+    return Collections.emptyMap();
+  }
+  
+  @Override
+  public List<TableSetup> getTablesToCreate() {
+    return Collections.emptyList();
+  }
+  
+  @Override
+  public void run() throws Exception {
+    getConnector().tableOperations().create("test_ingest");
+    
+    getConnector().tableOperations().addSplits("test_ingest", TestIngest.CreateTable.getSplitPoints(0,
100000, 50));
+    
+    TestIngest.main(new String[] {"-random", "89", "-timestamp", "7", "-size", "" + 50, "100000",
"0", "1"});
+    
+    getConnector().tableOperations().flush("test_ingest", null, null, true);
+    
+    List<Text> splits = new ArrayList<Text>(TestIngest.CreateTable.getSplitPoints(0,
100000, 67));
+    Random rand = new Random();
+    
+    for (int i = 0; i < 100; i++) {
+      int idx1 = rand.nextInt(splits.size() - 1);
+      int idx2 = rand.nextInt(splits.size() - (idx1 + 1)) + idx1 + 1;
+      
+      getConnector().tableOperations().compact("test_ingest", splits.get(idx1), splits.get(idx2),
false, false);
+    }
+    
+    getConnector().tableOperations().offline("test_ingest");
+  }
+  
+  public static void main(String[] args) throws Exception {
+    ArrayList<String> argsList = new ArrayList<String>();
+    argsList.addAll(Arrays.asList(args));
+    argsList.addAll(Arrays.asList(FateStarvationTest.class.getName(), "run"));
+    FunctionalTest.main(argsList.toArray(new String[0]));
+  }
+
+}

Added: accumulo/branches/1.4/test/system/auto/simple/fateStartvation.py
URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/test/system/auto/simple/fateStartvation.py?rev=1393868&view=auto
==============================================================================
--- accumulo/branches/1.4/test/system/auto/simple/fateStartvation.py (added)
+++ accumulo/branches/1.4/test/system/auto/simple/fateStartvation.py Thu Oct  4 01:27:15 2012
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from JavaTest import JavaTest
+
+import unittest
+
+class FateStarvationTest(JavaTest):
+    "Try to trigger a bug that was found in FATE"
+
+    order = 21
+    testClass="org.apache.accumulo.server.test.functional.FateStarvationTest"
+
+
+def suite():
+    result = unittest.TestSuite()
+    result.addTest(FateStarvationTest())
+    return result

Propchange: accumulo/branches/1.4/test/system/auto/simple/fateStartvation.py
------------------------------------------------------------------------------
    svn:executable = *



Mime
View raw message