accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [16/50] [abbrv] git commit: ACCUMULO-1537 started the stress tests, fixed some tests, made others more reliable
Date Wed, 17 Jul 2013 02:33:18 GMT
ACCUMULO-1537 started the stress tests, fixed some tests, made others more reliable

git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1499562 13f79535-47bb-0310-9956-ffa450edef68


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/390ca3f2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/390ca3f2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/390ca3f2

Branch: refs/heads/ACCUMULO-1496
Commit: 390ca3f2ebe75585ed8c97512e9985711f5d641d
Parents: 446a37a
Author: Eric C. Newton <ecn@apache.org>
Authored: Wed Jul 3 21:21:48 2013 +0000
Committer: Eric C. Newton <ecn@apache.org>
Committed: Wed Jul 3 21:21:48 2013 +0000

----------------------------------------------------------------------
 .../accumulo/test/TestMultiTableIngest.java     |   4 +-
 .../test/functional/CreateManyScannersTest.java |  51 ---------
 .../test/functional/BigRootTabletIT.java        |  63 +++++++++++
 .../accumulo/test/functional/BinaryIT.java      |   2 +-
 .../test/functional/BinaryStressIT.java         |  55 ++++++++++
 .../accumulo/test/functional/BloomFilterIT.java |  20 ++--
 .../test/functional/ChaoticBlancerIT.java       |  65 +++++++++++
 .../test/functional/CreateManyScannersIT.java   |  34 ++++++
 .../test/functional/FunctionalTestUtils.java    |   8 ++
 .../test/functional/MetadataMaxFiles.java       | 108 +++++++++++++++++++
 .../accumulo/test/functional/ReadWriteIT.java   |   2 +-
 .../accumulo/test/functional/SplitIT.java       |   8 +-
 .../accumulo/test/functional/TimeoutIT.java     |   2 +-
 .../accumulo/test/functional/WriteLotsIT.java   |  66 ++++++++++++
 test/system/auto/stress/batchWrite.py           |  77 -------------
 test/system/auto/stress/bigRootTablet.py        |  50 ---------
 test/system/auto/stress/binary.py               |  52 ---------
 test/system/auto/stress/manyScanners.py         |  31 ------
 test/system/auto/stress/metadataMaxFiles.py     |  66 ------------
 19 files changed, 419 insertions(+), 345 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java b/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
index b051988..1458065 100644
--- a/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
@@ -54,7 +54,7 @@ public class TestMultiTableIngest {
       scanner.setBatchSize(scanOpts.scanBatchSize);
       int count = i;
       for (Entry<Key,Value> elt : scanner) {
-        String expected = String.format("%05d", count);
+        String expected = String.format("%06d", count);
         if (!elt.getKey().getRow().toString().equals(expected))
           throw new RuntimeException("entry " + elt + " does not match expected " + expected
+ " in table " + table);
         count += tableNames.size();
@@ -96,7 +96,7 @@ public class TestMultiTableIngest {
       
       // populate
       for (int i = 0; i < opts.count; i++) {
-        Mutation m = new Mutation(new Text(String.format("%05d", i)));
+        Mutation m = new Mutation(new Text(String.format("%06d", i)));
         m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes()));
         b.getBatchWriter(tableNames.get(i % tableNames.size())).addMutation(m);
       }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/main/java/org/apache/accumulo/test/functional/CreateManyScannersTest.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/CreateManyScannersTest.java
b/test/src/main/java/org/apache/accumulo/test/functional/CreateManyScannersTest.java
deleted file mode 100644
index c84fb9b..0000000
--- a/test/src/main/java/org/apache/accumulo/test/functional/CreateManyScannersTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.functional;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.security.Authorizations;
-
-public class CreateManyScannersTest extends FunctionalTest {
-  
-  @Override
-  public void cleanup() throws Exception {
-    
-  }
-  
-  @Override
-  public Map<String,String> getInitialConfig() {
-    return Collections.emptyMap();
-  }
-  
-  @Override
-  public List<TableSetup> getTablesToCreate() {
-    return Collections.singletonList(new TableSetup("mscant"));
-  }
-  
-  @Override
-  public void run() throws Exception {
-    Connector connector = getConnector();
-    for (int i = 0; i < 100000; i++) {
-      connector.createScanner("mscant", Authorizations.EMPTY);
-    }
-  }
-  
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/BigRootTabletIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BigRootTabletIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BigRootTabletIT.java
new file mode 100644
index 0000000..308560f
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BigRootTabletIT.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.*;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.MetadataTable;
+import org.apache.accumulo.core.util.RootTable;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.junit.Test;
+
+public class BigRootTabletIT extends MacTest {
+  // ACCUMULO-542: A large root tablet will fail to load if it does't fit in the tserver
scan buffers
+  
+  @Override
+  public void configure(MiniAccumuloConfig cfg) {
+    Map<String,String> siteConfig = new HashMap<String, String>();
+    siteConfig.put(Property.TABLE_SCAN_MAXMEM.getKey(), "1024");
+    siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "60m");
+    cfg.setSiteConfig(siteConfig );
+  }
+
+  @Test(timeout=60*1000)
+  public void test() throws Exception {
+    Connector c = getConnector();
+    c.tableOperations().addSplits(MetadataTable.NAME, FunctionalTestUtils.splits("0 1 2 3
4 5 6 7 8 9 a".split(" ")));
+    for (int i = 0; i < 10; i++) {
+      c.tableOperations().create("" + i);
+      c.tableOperations().flush(MetadataTable.NAME, null, null, true);
+      c.tableOperations().flush(RootTable.NAME, null, null, true);
+    }
+    cluster.stop();
+    cluster.start();
+    int count = 0;
+    for (@SuppressWarnings("unused") Entry<Key,Value> entry : c.createScanner(RootTable.NAME,
Authorizations.EMPTY))
+      count++;
+    assertTrue(count > 0);
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/BinaryIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BinaryIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BinaryIT.java
index 484c55c..fb28715 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BinaryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BinaryIT.java
@@ -45,7 +45,7 @@ public class BinaryIT extends MacTest {
     runTest(c);
   }
   
-  void runTest(Connector c) throws Exception {
+  public static void runTest(Connector c) throws Exception {
     BatchWriterOpts bwOpts = new BatchWriterOpts();
     ScannerOpts scanOpts = new ScannerOpts();
     TestBinaryRows.Opts opts = new TestBinaryRows.Opts();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java
new file mode 100644
index 0000000..8c3af67
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+public class BinaryStressIT extends MacTest {
+  
+  @Override
+  public void configure(MiniAccumuloConfig cfg) {
+    Map<String,String> siteConfig = new HashMap<String,String>();
+    siteConfig.put(Property.TSERV_MAXMEM.getKey(), "50K");
+    siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "0");
+    cfg.setSiteConfig(siteConfig );
+  }
+
+  @Test(timeout=60*1000)
+  public void binaryStressTest() throws Exception {
+    Connector c = getConnector();
+    c.tableOperations().create("bt");
+    c.tableOperations().setProperty("bt", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
+    BinaryIT.runTest(c);
+    String id = c.tableOperations().tableIdMap().get("bt");
+    FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+    FileStatus[] dir = fs.listStatus(new Path(cluster.getConfig().getDir() + "/accumulo/tables/"
+ id));
+    assertTrue(dir.length  > 7);
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
index 19153df..d222991 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
@@ -49,15 +49,15 @@ public class BloomFilterIT extends MacTest {
     cfg.setDefaultMemory(500, MemoryUnit.MEGABYTE);
   }
   
-  @Test(timeout=120*1000)
+  @Test(timeout=500*1000)
   public void test() throws Exception {
     Connector c = getConnector();
     for (String table : "bt1 bt2 bt3 bt4".split(" ")) {
       c.tableOperations().create(table);
     }
-    write(c, "bt1", 1, 0, 1000000000, 500);
-    write(c, "bt2", 2, 0, 1000000000, 500);
-    write(c, "bt3", 3, 0, 1000000000, 500);
+    write(c, "bt1", 1, 0, 1000000000, 100);
+    write(c, "bt2", 2, 0, 1000000000, 100);
+    write(c, "bt3", 3, 0, 1000000000, 100);
     
     // test inserting an empty key
     BatchWriter bw = c.createBatchWriter("bt4", new BatchWriterConfig());
@@ -80,9 +80,9 @@ public class BloomFilterIT extends MacTest {
     FunctionalTestUtils.checkRFiles(c, "bt4", 1, 1, 1, 1);
     
     // these queries should only run quickly if bloom filters are working, so lets get a
base
-    long t1 = query(c, "bt1", 1, 0, 1000000000, 100000, 500);
-    long t2 = query(c, "bt2", 2, 0, 1000000000, 100000, 500);
-    long t3 = query(c, "bt3", 3, 0, 1000000000, 100000, 500);
+    long t1 = query(c, "bt1", 1, 0, 1000000000, 100000, 100);
+    long t2 = query(c, "bt2", 2, 0, 1000000000, 100000, 100);
+    long t3 = query(c, "bt3", 3, 0, 1000000000, 100000, 100);
     
     c.tableOperations().setProperty("bt1", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
     c.tableOperations().setProperty("bt1", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), RowFunctor.class.getName());
@@ -102,9 +102,9 @@ public class BloomFilterIT extends MacTest {
     
     // these queries should only run quickly if bloom
     // filters are working
-    long tb1 = query(c, "bt1", 1, 0, 1000000000, 100000, 500);
-    long tb2 = query(c, "bt2", 2, 0, 1000000000, 100000, 500);
-    long tb3 = query(c, "bt3", 3, 0, 1000000000, 100000, 500);
+    long tb1 = query(c, "bt1", 1, 0, 1000000000, 100000, 100);
+    long tb2 = query(c, "bt2", 2, 0, 1000000000, 100000, 100);
+    long tb3 = query(c, "bt3", 3, 0, 1000000000, 100000, 100);
     
     timeCheck(t1, tb1);
     timeCheck(t2, tb2);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/ChaoticBlancerIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ChaoticBlancerIT.java
b/test/src/test/java/org/apache/accumulo/test/functional/ChaoticBlancerIT.java
new file mode 100644
index 0000000..fb46f1e
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ChaoticBlancerIT.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.server.master.balancer.ChaoticLoadBalancer;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class ChaoticBlancerIT extends MacTest {
+  
+  @Override
+  public void configure(MiniAccumuloConfig cfg) {
+    Map<String,String> siteConfig = new HashMap<String, String>();
+    siteConfig.put(Property.TSERV_MAXMEM.getKey(), "10K");
+    siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "0");
+    siteConfig.put(Property.TABLE_LOAD_BALANCER.getKey(), ChaoticLoadBalancer.class.getName());
+    cfg.setSiteConfig(siteConfig );
+  }
+
+  @Test(timeout=120*1000)
+  public void test() throws Exception {
+    Connector c = getConnector();
+    c.tableOperations().create("test_ingest");
+    c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(),
"10K");
+    SortedSet<Text> splits = new TreeSet<Text>();
+    for (int i = 0; i < 200; i++) {
+      splits.add(new Text(String.format("%03d", i)));
+    }
+    c.tableOperations().create("unused");
+    c.tableOperations().addSplits("unused", splits);
+    TestIngest.Opts opts = new TestIngest.Opts();
+    VerifyIngest.Opts vopts = new VerifyIngest.Opts();
+    vopts.rows = opts.rows = 200000;
+    TestIngest.ingest(c, opts, new BatchWriterOpts());
+    c.tableOperations().flush("test_ingest", null, null, true);
+    VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/CreateManyScannersIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/CreateManyScannersIT.java
b/test/src/test/java/org/apache/accumulo/test/functional/CreateManyScannersIT.java
new file mode 100644
index 0000000..c41fae1
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CreateManyScannersIT.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.security.Authorizations;
+import org.junit.Test;
+
+public class CreateManyScannersIT extends MacTest {
+  
+  @Test(timeout=10*1000)
+  public void run() throws Exception {
+    Connector c = getConnector();
+    c.tableOperations().create("mscant");
+    for (int i = 0; i < 100000; i++) {
+      c.createScanner("mscant", Authorizations.EMPTY);
+    }
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
b/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
index 4af2c96..8fd50f5 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
@@ -23,8 +23,10 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.SortedSet;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -160,5 +162,11 @@ public class FunctionalTestUtils {
   static Mutation nm(String row, String cf, String cq, String value) {
     return nm(row, cf, cq, new Value(value.getBytes()));
   }
+  public static SortedSet<Text> splits(String [] splits) {
+    SortedSet<Text> result = new TreeSet<Text>();
+    for (String split : splits)
+      result.add(new Text(split));
+    return result;
+  }
   
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFiles.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFiles.java
b/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFiles.java
new file mode 100644
index 0000000..ae503de
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFiles.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.*;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.master.thrift.MasterClientService.Client;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.MetadataTable;
+import org.apache.accumulo.core.util.RootTable;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.server.util.Admin;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class MetadataMaxFiles extends MacTest {
+  
+  @Override
+  public void configure(MiniAccumuloConfig cfg) {
+    Map<String,String> siteConfig = new HashMap<String, String>();
+    siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
+    siteConfig.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "10");
+    cfg.setSiteConfig(siteConfig );
+  }
+
+  @Test(timeout=240*1000)
+  public void test() throws Exception {
+    Connector c = getConnector();
+    SortedSet<Text> splits = new TreeSet<Text>();
+    for (int i = 0; i < 1000; i++) {
+      splits.add(new Text(String.format("%03d", i)));
+    }
+    c.tableOperations().setProperty(MetadataTable.NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(),
"10000");
+    for (int i = 0; i < 5; i++) {
+      String tableName = "table" + i;
+      log.info("Creating " + tableName);
+      c.tableOperations().create(tableName);
+      log.info("adding splits");
+      c.tableOperations().addSplits(tableName, splits);
+      log.info("flushing");
+      c.tableOperations().flush(MetadataTable.NAME, null, null, true);
+      c.tableOperations().flush(RootTable.NAME, null, null, true);
+    }
+    UtilWaitThread.sleep(20*1000);
+    log.info("shutting down");
+    assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+    cluster.stop();
+    log.info("starting up");
+    cluster.start();
+
+    UtilWaitThread.sleep(30*1000);
+
+    while (true) {
+      MasterMonitorInfo stats = null;
+      TCredentials creds = CredentialHelper.create("root", new PasswordToken(MacTest.PASSWORD),
c.getInstance().getInstanceName());
+      Client client = null;
+      try {
+        client = MasterClient.getConnectionWithRetry(c.getInstance());
+        stats = client.getMasterStats(Tracer.traceInfo(), creds);
+      } finally {
+        if (client != null)
+          MasterClient.close(client);
+      }
+      int tablets = 0;
+      for (TabletServerStatus tserver : stats.tServerInfo) {
+        for (Entry<String,TableInfo> entry : tserver.tableMap.entrySet()) {
+          if (entry.getKey().startsWith("!"))
+            continue;
+          tablets += entry.getValue().onlineTablets;
+        }
+      }
+      if (tablets == 5005)
+        break;
+      UtilWaitThread.sleep(1000);
+    }
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java b/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java
index bc89e20..7c74326 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java
@@ -57,7 +57,7 @@ import org.junit.Test;
 
 public class ReadWriteIT extends MacTest {
   
-  static final int ROWS = 20000;
+  static final int ROWS = 200000;
   static final int COLS = 1;
   static final String COLF = "colf";
   

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java b/test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java
index 741e216..c4719e0 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java
@@ -48,7 +48,7 @@ public class SplitIT extends MacTest {
   public void configure(MiniAccumuloConfig cfg) {
     Map<String,String> siteConfig = new HashMap<String,String>();
     siteConfig.put(Property.TSERV_MAXMEM.getKey(), "5K");
-    siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "1s");
+    siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "100ms");
     cfg.setSiteConfig(siteConfig);
   }
   
@@ -92,8 +92,8 @@ public class SplitIT extends MacTest {
     c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(),
"10K");
     c.tableOperations().setProperty("test_ingest", Property.TABLE_FILE_COMPRESSION_TYPE.getKey(),
"none");
     ReadWriteIT.interleaveTest(c);
-    UtilWaitThread.sleep(5 * 1000);
-    assertTrue(c.tableOperations().listSplits("test_ingest").size() > 10);
+    UtilWaitThread.sleep(5*1000);
+    assertTrue(c.tableOperations().listSplits("test_ingest").size() > 20);
   }
   
   @Test(timeout = 120 * 1000)
@@ -102,6 +102,8 @@ public class SplitIT extends MacTest {
     c.tableOperations().create("test_ingest");
     c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(),
"10K");
     DeleteIT.deleteTest(c);
+    c.tableOperations().flush("test_ingest", null, null, true);
+    UtilWaitThread.sleep(5*1000);
     assertTrue(c.tableOperations().listSplits("test_ingest").size() > 30);
   }
   

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/TimeoutIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/TimeoutIT.java b/test/src/test/java/org/apache/accumulo/test/functional/TimeoutIT.java
index 2dc6802..2c3c86d 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/TimeoutIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/TimeoutIT.java
@@ -42,7 +42,7 @@ import org.junit.Test;
  */
 public class TimeoutIT extends MacTest {
   
-  @Test
+  @Test(timeout=30*1000)
   public void run() throws Exception {
     Connector conn = getConnector();
     testBatchWriterTimeout(conn);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/WriteLotsIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/WriteLotsIT.java b/test/src/test/java/org/apache/accumulo/test/functional/WriteLotsIT.java
new file mode 100644
index 0000000..df8e656
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/WriteLotsIT.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.junit.Test;
+
+public class WriteLotsIT extends MacTest {
+  
+  @Test(timeout=20*1000)
+  public void writeLots() throws Exception {
+    final Connector c = getConnector();
+    c.tableOperations().create("test_ingest");
+    final AtomicReference<Exception> ref = new AtomicReference<Exception>();
+    List<Thread> threads = new ArrayList<Thread>();
+    for (int i = 0; i < 10; i++) {
+      final int index = i;
+      Thread t = new Thread() {
+        public void run() {
+          try {
+            TestIngest.Opts opts = new TestIngest.Opts();
+            opts.startRow = index * 10000;
+            opts.rows = 10000;
+            TestIngest.ingest(c, opts, new BatchWriterOpts());
+          } catch (Exception ex) {
+            ref.set(ex);
+          }
+        }
+      };
+      t.start();
+      threads.add(t);
+    }
+    for (Thread thread : threads) {
+      thread.join();
+    }
+    if (ref.get() != null) {
+      throw ref.get();
+    }
+    VerifyIngest.Opts vopts = new VerifyIngest.Opts();
+    vopts.rows = 10000 * 10;
+    VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/system/auto/stress/batchWrite.py
----------------------------------------------------------------------
diff --git a/test/system/auto/stress/batchWrite.py b/test/system/auto/stress/batchWrite.py
deleted file mode 100755
index f2f83b0..0000000
--- a/test/system/auto/stress/batchWrite.py
+++ /dev/null
@@ -1,77 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-import logging
-import unittest
-import time
-
-from TestUtils import TestUtilsMixin, ROOT_PASSWORD
-
-log = logging.getLogger('test.auto')
-
-class WriteLots(unittest.TestCase, TestUtilsMixin):
-    """Start a clean accumulo, ingest some data using lots of clients at once,
-    and verify it was stored properly"""
-
-    order = 30
-    settings = TestUtilsMixin.settings.copy()
-    settings['instance.zookeeper.timeout'] = '40s'
-
-    def ingest(self, host, start, count, **kwargs):
-        klass = 'org.apache.accumulo.test.TestIngest'
-        args = '--user root --size 50 --random 56 --rows %d --start %d --cols 1 -p %s' %
(count, start, ROOT_PASSWORD)
-        return self.runClassOn(host, klass, args.split(), **kwargs)
-
-    def setUp(self):
-        TestUtilsMixin.setUp(self);
-        
-        # initialize the database
-        self.createTable("test_ingest")
-
-    def tearDown(self):
-        TestUtilsMixin.tearDown(self)
-        self.pkill(self.masterHost(), 'TestIngest')
-
-    def runTest(self):
-        N = 10*len(self.hosts)
-        waitTime = 60 * N * self.options.rows / 200000 + 90
-
-        log.info("Starting %d clients", N)
-        handles = []
-        for i in range(N):
-            # start test ingestion
-            handle = self.ingest(self.hosts[i % len(self.hosts)],
-                                 i * self.options.rows,
-				 self.options.rows)
-            handles.append(handle)
-
-        end = time.time() + waitTime
-        for handle in handles:
-            waitTime = end - time.time()
-            log.debug("Waiting %s seconds", waitTime)
-            self.waitForStop(handle, waitTime)
-
-        log.info("Verifying Ingestion")
-        self.waitForStop(self.verify(self.masterHost(), self.options.rows * N),
-                         waitTime)
-        self.shutdown_accumulo()
-        
-
-def suite():
-    result = unittest.TestSuite()
-    result.addTest(WriteLots())
-    return result

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/system/auto/stress/bigRootTablet.py
----------------------------------------------------------------------
diff --git a/test/system/auto/stress/bigRootTablet.py b/test/system/auto/stress/bigRootTablet.py
deleted file mode 100755
index d69a79c..0000000
--- a/test/system/auto/stress/bigRootTablet.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import logging
-import unittest
-
-from TestUtils import TestUtilsMixin
-
-log = logging.getLogger('test.auto')
-
-class BigRootTablet(TestUtilsMixin, unittest.TestCase):
-    "ACCUMULO-542: A large root tablet will fail to load if it does't fit in the tserver
scan buffers"
-
-    order = 80
-
-    settings = TestUtilsMixin.settings.copy()
-    settings['table.scan.max.memory'] = '1024'
-    settings['tserver.compaction.major.delay'] = '60m'
-
-    def setUp(self):
-        TestUtilsMixin.setUp(self);
-    
-    def tearDown(self):
-        TestUtilsMixin.tearDown(self);
-    
-    def runTest(self):
-	cmd = 'table !METADATA\naddsplits 0 1 2 3 4 5 6 7 8 9 a\n'
-        for i in range(10):
-	    cmd += 'createtable %s\nflush -t !METADATA\n' % i
-        self.shell(self.masterHost(), cmd)
-	self.stop_accumulo()
-	self.start_accumulo()
-
-def suite():
-    result = unittest.TestSuite()
-    result.addTest(BigRootTablet())
-    return result

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/system/auto/stress/binary.py
----------------------------------------------------------------------
diff --git a/test/system/auto/stress/binary.py b/test/system/auto/stress/binary.py
deleted file mode 100755
index 41f584d..0000000
--- a/test/system/auto/stress/binary.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import unittest
-
-import os
-import logging
-
-from TestUtils import ACCUMULO_DIR
-from simple.binary import BinaryTest
-
-log = logging.getLogger('test.auto')
-
-class BinaryStressTest(BinaryTest) :
-    order = 80
-
-    tableSettings = BinaryTest.tableSettings.copy()
-    tableSettings['bt'] = { 
-    	'table.split.threshold': '10K',
-        }
-    settings = BinaryTest.settings.copy()
-    settings.update({
-        'tserver.memory.maps.max':'50K',
-        'tserver.compaction.major.delay': 0,
-        })
-
-    def runTest(self):
-        BinaryTest.runTest(self)
-        handle = self.runOn(self.masterHost(), [
-            'hadoop', 'fs', '-ls', os.path.join(ACCUMULO_DIR,'tables',self.getTableId('bt'))
-            ])
-        out, err = handle.communicate()
-        if len(out.split('\n')) < 8:
-            log.debug(out)
-        self.assert_(len(out.split('\n')) > 7)
-
-def suite():
-    result = unittest.TestSuite()
-    result.addTest(BinaryStressTest())
-    return result

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/system/auto/stress/manyScanners.py
----------------------------------------------------------------------
diff --git a/test/system/auto/stress/manyScanners.py b/test/system/auto/stress/manyScanners.py
deleted file mode 100755
index 229b931..0000000
--- a/test/system/auto/stress/manyScanners.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from JavaTest import JavaTest
-
-import unittest
-
-class CreateManyScannersTest(JavaTest):
-    "Test creating a lot of scanners"
-
-    order = 9999
-    testClass="org.apache.accumulo.test.functional.CreateManyScannersTest"
-    maxRuntime = 60
-
-
-def suite():
-    result = unittest.TestSuite()
-    result.addTest(CreateManyScannersTest())
-    return result

http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/system/auto/stress/metadataMaxFiles.py
----------------------------------------------------------------------
diff --git a/test/system/auto/stress/metadataMaxFiles.py b/test/system/auto/stress/metadataMaxFiles.py
deleted file mode 100755
index 8a65b81..0000000
--- a/test/system/auto/stress/metadataMaxFiles.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import unittest
-
-from TestUtils import TestUtilsMixin
-import os
-
-import logging
-log = logging.getLogger('test.auto')
-
-class MetadataMaxFiles(TestUtilsMixin, unittest.TestCase):
-    "open a large !METADATA with too few files"
-
-    order=75
-
-    settings = TestUtilsMixin.settings.copy()
-    settings['tserver.compaction.major.delay'] = 1
-
-    def runTest(self):
-        # Create a bunch of tables with splits to split the !METADATA table
-        self.splitfile = 'splits'
-        fp = open(self.splitfile, 'w')
-        for i in range(1000):
-            fp.write('%03d\n' % i)
-        fp.close()
-        self.splitfile = os.path.realpath(self.splitfile)
-        self.shell(self.masterHost(),
-                   'config -t !METADATA -s table.split.threshold=10000\n' + 
-                   ''.join(['createtable test%d -sf %s\nflush -t !METADATA\n' % (i, self.splitfile)
for i in range(5)]))
-        self.shutdown_accumulo(150)
-        
-        # reconfigure accumulo to use a very small number of files
-        self.stop_accumulo()
-        self.settings['tserver.scan.files.open.max'] = 10
-        self.create_config_file(self.settings)
-
-        # make sure the master knows about all the tables we created
-        self.sleep(5)
-        self.start_accumulo()
-        self.sleep(60)
-        h = self.runOn(self.masterHost(),
-                       [self.accumulo_sh(), 'org.apache.accumulo.test.GetMasterStats'])
-        out, err = h.communicate()
-        self.assert_(len([x for x in out.split('\n') if x.find('  Tablets: 1001') == 0])
== 5)
-
-    def tearDown(self):
-        TestUtilsMixin.tearDown(self)
-        os.unlink(self.splitfile)
-
-def suite():
-    result = unittest.TestSuite()
-    result.addTest(MetadataMaxFiles())
-    return result


Mime
View raw message