accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [2/3] accumulo git commit: Merge branch '1.7' into 1.8
Date Wed, 25 Jan 2017 17:54:37 GMT
Merge branch '1.7' into 1.8

Conflicts:
	server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b903766e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b903766e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b903766e

Branch: refs/heads/master
Commit: b903766ec14b36ac4185ab658d2155970cc7ffb4
Parents: 95be9f3 df400c5
Author: Keith Turner <kturner@apache.org>
Authored: Wed Jan 25 12:42:41 2017 -0500
Committer: Keith Turner <kturner@apache.org>
Committed: Wed Jan 25 12:42:41 2017 -0500

----------------------------------------------------------------------
 .../org/apache/accumulo/fate/AdminUtil.java     | 104 +++++++++++--
 .../accumulo/cluster/AccumuloCluster.java       |   6 +
 .../standalone/StandaloneAccumuloCluster.java   |  12 ++
 .../impl/MiniAccumuloClusterImpl.java           |   8 +
 .../accumulo/master/FateServiceHandler.java     |   2 +-
 .../accumulo/master/tableOps/DeleteTable.java   |  49 +++----
 .../apache/accumulo/master/tableOps/Utils.java  |  13 ++
 .../test/functional/BackupMasterIT.java         |   7 +-
 .../functional/ConcurrentDeleteTableIT.java     | 147 +++++++++++++++++++
 9 files changed, 307 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/b903766e/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
index a3180a7,79ad527..829b321
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
@@@ -110,12 -111,12 +111,13 @@@ import org.slf4j.LoggerFactory
  import com.google.common.annotations.VisibleForTesting;
  import com.google.common.base.Joiner;
  import com.google.common.base.Predicate;
+ import com.google.common.collect.Iterables;
  import com.google.common.collect.Maps;
 +import com.google.common.util.concurrent.Uninterruptibles;
  
  /**
 - * A utility class that will create Zookeeper and Accumulo processes that write all of their
data to a single local directory. This class makes it easy to test
 - * code against a real Accumulo instance. Its much more accurate for testing than {@link
org.apache.accumulo.core.client.mock.MockAccumulo}, but much slower.
 + * This class provides the backing implementation for {@link MiniAccumuloCluster}, and may
contain features for internal testing which have not yet been
 + * promoted to the public API. It's best to use {@link MiniAccumuloCluster} whenever possible.
Use of this class risks API breakage between versions.
   *
   * @since 1.6.0
   */

http://git-wip-us.apache.org/repos/asf/accumulo/blob/b903766e/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/b903766e/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
index 2baf7ac,9b921e2..aa02559
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
@@@ -117,6 -116,19 +117,19 @@@ public class Utils 
        return 100;
    }
  
+   public static String getNamespaceId(Instance instance, String tableId, TableOperation
op) throws Exception {
+     try {
+       return Tables.getNamespaceId(instance, tableId);
+     } catch (RuntimeException e) {
+       // see if this was caused because the table does not exists
+       IZooReaderWriter zk = ZooReaderWriter.getInstance();
+       if (!zk.exists(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId))
 -        throw new ThriftTableOperationException(tableId, "", op, TableOperationExceptionType.NOTFOUND,
"Table does not exist");
++        throw new AcceptableThriftTableOperationException(tableId, "", op, TableOperationExceptionType.NOTFOUND,
"Table does not exist");
+       else
+         throw e;
+     }
+   }
+ 
    public static long reserveHdfsDirectory(String directory, long tid) throws KeeperException,
InterruptedException {
      Instance instance = HdfsZooInstance.getInstance();
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/b903766e/test/src/main/java/org/apache/accumulo/test/functional/BackupMasterIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/BackupMasterIT.java
index d8979db,0000000..12336bd
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/BackupMasterIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BackupMasterIT.java
@@@ -1,68 -1,0 +1,71 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
 +import java.util.Collections;
 +import java.util.List;
 +
++import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.fate.util.UtilWaitThread;
- import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
++import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.master.Master;
++import org.apache.accumulo.server.zookeeper.ZooReaderWriterFactory;
 +import org.junit.Test;
 +
 +public class BackupMasterIT extends ConfigurableMacBase {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 120;
 +  }
 +
 +  @Test
 +  public void test() throws Exception {
 +    // wait for master
 +    UtilWaitThread.sleep(1000);
 +    // create a backup
 +    Process backup = exec(Master.class);
 +    try {
-       ZooReaderWriter writer = new ZooReaderWriter(cluster.getZooKeepers(), 30 * 1000, "digest",
"accumulo:DONTTELL".getBytes());
++      String secret = getCluster().getSiteConfiguration().get(Property.INSTANCE_SECRET);
++      IZooReaderWriter writer = new ZooReaderWriterFactory().getZooReaderWriter(cluster.getZooKeepers(),
30 * 1000, secret);
 +      String root = "/accumulo/" + getConnector().getInstance().getInstanceID();
 +      List<String> children = Collections.emptyList();
 +      // wait for 2 lock entries
 +      do {
 +        UtilWaitThread.sleep(100);
 +        children = writer.getChildren(root + "/masters/lock");
 +      } while (children.size() != 2);
 +      Collections.sort(children);
 +      // wait for the backup master to learn to be the backup
 +      UtilWaitThread.sleep(1000);
 +      // generate a false zookeeper event
 +      String lockPath = root + "/masters/lock/" + children.get(0);
 +      byte data[] = writer.getData(lockPath, null);
 +      writer.getZooKeeper().setData(lockPath, data, -1);
 +      // let it propagate
 +      UtilWaitThread.sleep(500);
 +      // kill the master by removing its lock
 +      writer.recursiveDelete(lockPath, NodeMissingPolicy.FAIL);
 +      // ensure the backup becomes the master
 +      getConnector().tableOperations().create(getUniqueNames(1)[0]);
 +    } finally {
 +      backup.destroy();
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/b903766e/test/src/main/java/org/apache/accumulo/test/functional/ConcurrentDeleteTableIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/ConcurrentDeleteTableIT.java
index 0000000,0000000..3f7a305
new file mode 100644
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ConcurrentDeleteTableIT.java
@@@ -1,0 -1,0 +1,147 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one or more
++ * contributor license agreements.  See the NOTICE file distributed with
++ * this work for additional information regarding copyright ownership.
++ * The ASF licenses this file to You under the Apache License, Version 2.0
++ * (the "License"); you may not use this file except in compliance with
++ * the License.  You may obtain a copy of the License at
++ *
++ *     http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.accumulo.test.functional;
++
++import java.util.ArrayList;
++import java.util.List;
++import java.util.Random;
++import java.util.TreeSet;
++import java.util.concurrent.CountDownLatch;
++import java.util.concurrent.ExecutorService;
++import java.util.concurrent.Executors;
++import java.util.concurrent.Future;
++
++import org.apache.accumulo.core.Constants;
++import org.apache.accumulo.core.client.AccumuloException;
++import org.apache.accumulo.core.client.AccumuloSecurityException;
++import org.apache.accumulo.core.client.BatchWriter;
++import org.apache.accumulo.core.client.BatchWriterConfig;
++import org.apache.accumulo.core.client.Connector;
++import org.apache.accumulo.core.client.Instance;
++import org.apache.accumulo.core.client.MutationsRejectedException;
++import org.apache.accumulo.core.client.TableNotFoundException;
++import org.apache.accumulo.core.conf.Property;
++import org.apache.accumulo.core.data.Mutation;
++import org.apache.accumulo.core.security.Authorizations;
++import org.apache.accumulo.core.zookeeper.ZooUtil;
++import org.apache.accumulo.fate.AdminUtil;
++import org.apache.accumulo.fate.AdminUtil.FateStatus;
++import org.apache.accumulo.fate.ZooStore;
++import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
++import org.apache.accumulo.harness.AccumuloClusterHarness;
++import org.apache.accumulo.server.zookeeper.ZooReaderWriterFactory;
++import org.apache.hadoop.io.Text;
++import org.apache.zookeeper.KeeperException;
++import org.junit.Assert;
++import org.junit.Test;
++
++public class ConcurrentDeleteTableIT extends AccumuloClusterHarness {
++
++  @Test
++  public void testConcurrentDeleteTablesOps() throws Exception {
++    final Connector c = getConnector();
++    String[] tables = getUniqueNames(2);
++
++    TreeSet<Text> splits = new TreeSet<>();
++
++    for (int i = 0; i < 1000; i++) {
++      Text split = new Text(String.format("%09x", i * 100000));
++      splits.add(split);
++    }
++
++    ExecutorService es = Executors.newFixedThreadPool(20);
++
++    int count = 0;
++    for (final String table : tables) {
++      c.tableOperations().create(table);
++      c.tableOperations().addSplits(table, splits);
++      writeData(c, table);
++      if (count == 1) {
++        c.tableOperations().flush(table, null, null, true);
++      }
++      count++;
++
++      final CountDownLatch cdl = new CountDownLatch(20);
++
++      List<Future<?>> futures = new ArrayList<>();
++
++      for (int i = 0; i < 20; i++) {
++        Future<?> future = es.submit(new Runnable() {
++
++          @Override
++          public void run() {
++            try {
++              cdl.countDown();
++              cdl.await();
++              c.tableOperations().delete(table);
++            } catch (TableNotFoundException e) {
++              // expected
++            } catch (InterruptedException | AccumuloException | AccumuloSecurityException
e) {
++              throw new RuntimeException(e);
++            }
++          }
++        });
++
++        futures.add(future);
++      }
++
++      for (Future<?> future : futures) {
++        future.get();
++      }
++
++      try {
++        c.createScanner(table, Authorizations.EMPTY);
++        Assert.fail("Expected table " + table + " to be gone.");
++      } catch (TableNotFoundException tnfe) {
++        // expected
++      }
++
++      FateStatus fateStatus = getFateStatus();
++
++      // ensure there are no dangling locks... before ACCUMULO-4575 was fixed concurrent
delete tables could fail and leave dangling locks.
++      Assert.assertEquals(0, fateStatus.getDanglingHeldLocks().size());
++      Assert.assertEquals(0, fateStatus.getDanglingWaitingLocks().size());
++    }
++
++    es.shutdown();
++  }
++
++  private FateStatus getFateStatus() throws KeeperException, InterruptedException {
++    Instance instance = getConnector().getInstance();
++    AdminUtil<String> admin = new AdminUtil<>(false);
++    String secret = getCluster().getSiteConfiguration().get(Property.INSTANCE_SECRET);
++    IZooReaderWriter zk = new ZooReaderWriterFactory().getZooReaderWriter(instance.getZooKeepers(),
instance.getZooKeepersSessionTimeOut(), secret);
++    ZooStore<String> zs = new ZooStore<String>(ZooUtil.getRoot(instance) + Constants.ZFATE,
zk);
++    FateStatus fateStatus = admin.getStatus(zs, zk, ZooUtil.getRoot(instance) + Constants.ZTABLE_LOCKS,
null, null);
++    return fateStatus;
++  }
++
++  private void writeData(Connector c, String table) throws TableNotFoundException, MutationsRejectedException
{
++    BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig());
++    try {
++      Random rand = new Random();
++      for (int i = 0; i < 1000; i++) {
++        Mutation m = new Mutation(String.format("%09x", rand.nextInt(100000 * 1000)));
++        m.put("m", "order", "" + i);
++        bw.addMutation(m);
++      }
++    } finally {
++      bw.close();
++    }
++  }
++}


Mime
View raw message