Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3DB3A10951 for ; Thu, 12 Dec 2013 16:25:40 +0000 (UTC) Received: (qmail 201 invoked by uid 500); 12 Dec 2013 16:25:35 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 99723 invoked by uid 500); 12 Dec 2013 16:25:27 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 99257 invoked by uid 99); 12 Dec 2013 16:25:23 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 Dec 2013 16:25:23 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 999BF89FB87; Thu, 12 Dec 2013 16:25:23 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ecn@apache.org To: commits@accumulo.apache.org Date: Thu, 12 Dec 2013 16:25:35 -0000 Message-Id: <3a10332f48eb490a9ab6b3b95dae5ffe@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [13/16] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7eb838e3 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7eb838e3 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7eb838e3 Branch: refs/heads/1.5.1-SNAPSHOT Commit: 7eb838e3cc6c2d0bb5491d101cbb2c0bd67e653e Parents: 994e43c 0d0bc46 Author: Eric Newton Authored: Thu Dec 12 11:24:14 2013 -0500 Committer: Eric Newton Committed: Thu Dec 12 11:24:14 2013 -0500 ---------------------------------------------------------------------- .../org/apache/accumulo/core/client/Instance.java | 7 ++++--- .../accumulo/core/client/ZooKeeperInstance.java | 6 +++--- .../accumulo/core/client/mock/MockInstance.java | 2 +- .../core/client/impl/TabletLocatorImplTest.java | 2 +- .../org/apache/accumulo/fate/zookeeper/ZooCache.java | 6 ++++-- .../apache/accumulo/fate/zookeeper/ZooReader.java | 15 ++++++++++++--- .../accumulo/server/client/HdfsZooInstance.java | 8 ++------ 7 files changed, 27 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/7eb838e3/core/src/main/java/org/apache/accumulo/core/client/Instance.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/client/Instance.java index 612301e,0000000..6449765 mode 100644,000000..100644 --- a/core/src/main/java/org/apache/accumulo/core/client/Instance.java +++ b/core/src/main/java/org/apache/accumulo/core/client/Instance.java @@@ -1,173 -1,0 +1,174 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.client; + ++import java.io.Closeable; +import java.nio.ByteBuffer; +import java.util.List; + +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.conf.AccumuloConfiguration; + +/** + * This class represents the information a client needs to know to connect to an instance of accumulo. + * + */ - public interface Instance { ++public interface Instance extends Closeable { + /** + * Returns the location of the tablet server that is serving the root tablet. + * + * @return location in "hostname:port" form + */ + public abstract String getRootTabletLocation(); + + /** + * Returns the location(s) of the accumulo master and any redundant servers. + * + * @return a list of locations in "hostname:port" form + */ + public abstract List getMasterLocations(); + + /** + * Returns a unique string that identifies this instance of accumulo. + * + * @return a UUID + */ + public abstract String getInstanceID(); + + /** + * Returns the instance name given at system initialization time. + * + * @return current instance name + */ + public abstract String getInstanceName(); + + /** + * Returns a comma-separated list of zookeeper servers the instance is using. + * + * @return the zookeeper servers this instance is using in "hostname:port" form + */ + public abstract String getZooKeepers(); + + /** + * Returns the zookeeper connection timeout. + * + * @return the configured timeout to connect to zookeeper + */ + public abstract int getZooKeepersSessionTimeOut(); + + /** + * Returns a connection to accumulo. + * + * @param user + * a valid accumulo user + * @param pass + * A UTF-8 encoded password. The password may be cleared after making this call. + * @return the accumulo Connector + * @throws AccumuloException + * when a generic exception occurs + * @throws AccumuloSecurityException + * when a user's credentials are invalid + * @deprecated since 1.5, use {@link #getConnector(String, AuthenticationToken)} with {@link PasswordToken} + */ + @Deprecated + public abstract Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException; + + /** + * Returns a connection to accumulo. + * + * @param auth + * An Credentials object. + * @return the accumulo Connector + * @throws AccumuloException + * when a generic exception occurs + * @throws AccumuloSecurityException + * when a user's credentials are invalid + * @deprecated since 1.5, use {@link #getConnector(String, AuthenticationToken)} with {@link PasswordToken} + */ + @Deprecated + public abstract Connector getConnector(org.apache.accumulo.core.security.thrift.AuthInfo auth) throws AccumuloException, AccumuloSecurityException; + + /** + * Returns a connection to accumulo. + * + * @param user + * a valid accumulo user + * @param pass + * A UTF-8 encoded password. The password may be cleared after making this call. + * @return the accumulo Connector + * @throws AccumuloException + * when a generic exception occurs + * @throws AccumuloSecurityException + * when a user's credentials are invalid + * @deprecated since 1.5, use {@link #getConnector(String, AuthenticationToken)} with {@link PasswordToken} + */ + @Deprecated + public abstract Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException; + + /** + * Returns a connection to this instance of accumulo. + * + * @param user + * a valid accumulo user + * @param pass + * If a mutable CharSequence is passed in, it may be cleared after this call. + * @return the accumulo Connector + * @throws AccumuloException + * when a generic exception occurs + * @throws AccumuloSecurityException + * when a user's credentials are invalid + * @deprecated since 1.5, use {@link #getConnector(String, AuthenticationToken)} with {@link PasswordToken} + */ + @Deprecated + public abstract Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException; + + /** + * Closes up the instance to free up all associated resources. You should try to reuse an Instance as much as you can because there is some location caching + * stored which will enhance performance. - * @throws AccumuloException + */ - public abstract void close() throws AccumuloException; ++ @Override ++ public abstract void close(); + + /** + * Returns the AccumuloConfiguration to use when interacting with this instance. + * + * @return the AccumuloConfiguration that specifies properties related to interacting with this instance + */ + public abstract AccumuloConfiguration getConfiguration(); + + /** + * Set the AccumuloConfiguration to use when interacting with this instance. + * + * @param conf + * accumulo configuration + */ + public abstract void setConfiguration(AccumuloConfiguration conf); + + /** + * Returns a connection to this instance of accumulo. + * + * @param principal + * a valid accumulo user + * @param token + * Use the token type configured for the Accumulo instance you are connecting to. An Accumulo instance with default configurations will use + * {@link PasswordToken} + * @since 1.5.0 + */ + public abstract Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException; + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/7eb838e3/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java index c260947,0000000..6e1b660 mode 100644,000000..100644 --- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java +++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java @@@ -1,352 -1,0 +1,352 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.client; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.impl.ConnectorImpl; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.file.FileUtil; +import org.apache.accumulo.core.security.CredentialHelper; +import org.apache.accumulo.core.security.thrift.TCredentials; +import org.apache.accumulo.core.util.ArgumentChecker; +import org.apache.accumulo.core.util.ByteBufferUtil; +import org.apache.accumulo.core.util.CachedConfiguration; +import org.apache.accumulo.core.util.OpTimer; +import org.apache.accumulo.core.util.TextUtil; +import org.apache.accumulo.core.util.ThriftUtil; +import org.apache.accumulo.core.zookeeper.ZooUtil; +import org.apache.accumulo.fate.zookeeper.ZooCache; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + *

+ * An implementation of instance that looks in zookeeper to find information needed to connect to an instance of accumulo. + * + *

+ * The advantage of using zookeeper to obtain information about accumulo is that zookeeper is highly available, very responsive, and supports caching. + * + *

+ * Because it is possible for multiple instances of accumulo to share a single set of zookeeper servers, all constructors require an accumulo instance name. + * + * If you do not know the instance names then run accumulo org.apache.accumulo.server.util.ListInstances on an accumulo server. + * + */ + +public class ZooKeeperInstance implements Instance { + + private static final Logger log = Logger.getLogger(ZooKeeperInstance.class); + + private String instanceId = null; + private String instanceName = null; + + private final ZooCache zooCache; + + private final String zooKeepers; + + private final int zooKeepersSessionTimeOut; + + private volatile boolean closed = false; + + /** + * + * @param instanceName + * The name of specific accumulo instance. This is set at initialization time. + * @param zooKeepers + * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port. + */ + + public ZooKeeperInstance(String instanceName, String zooKeepers) { + this(instanceName, zooKeepers, (int) AccumuloConfiguration.getDefaultConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT)); + } + + /** + * + * @param instanceName + * The name of specific accumulo instance. This is set at initialization time. + * @param zooKeepers + * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port. + * @param sessionTimeout + * zoo keeper session time out in milliseconds. + */ + + public ZooKeeperInstance(String instanceName, String zooKeepers, int sessionTimeout) { + ArgumentChecker.notNull(instanceName, zooKeepers); + this.instanceName = instanceName; + this.zooKeepers = zooKeepers; + this.zooKeepersSessionTimeOut = sessionTimeout; + zooCache = ZooCache.getInstance(zooKeepers, sessionTimeout); + getInstanceID(); + clientInstances.incrementAndGet(); + } + + /** + * + * @param instanceId + * The UUID that identifies the accumulo instance you want to connect to. + * @param zooKeepers + * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port. + */ + + public ZooKeeperInstance(UUID instanceId, String zooKeepers) { + this(instanceId, zooKeepers, (int) AccumuloConfiguration.getDefaultConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT)); + } + + /** + * + * @param instanceId + * The UUID that identifies the accumulo instance you want to connect to. + * @param zooKeepers + * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port. + * @param sessionTimeout + * zoo keeper session time out in milliseconds. + */ + + public ZooKeeperInstance(UUID instanceId, String zooKeepers, int sessionTimeout) { + ArgumentChecker.notNull(instanceId, zooKeepers); + this.instanceId = instanceId.toString(); + this.zooKeepers = zooKeepers; + this.zooKeepersSessionTimeOut = sessionTimeout; + zooCache = ZooCache.getInstance(zooKeepers, sessionTimeout); + clientInstances.incrementAndGet(); + } + + @Override + public String getInstanceID() { + if (closed) + throw new RuntimeException("ZooKeeperInstance has been closed."); + if (instanceId == null) { + // want the instance id to be stable for the life of this instance object, + // so only get it once + String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + instanceName; + byte[] iidb = zooCache.get(instanceNamePath); + if (iidb == null) { + throw new RuntimeException("Instance name " + instanceName + + " does not exist in zookeeper. Run \"accumulo org.apache.accumulo.server.util.ListInstances\" to see a list."); + } + instanceId = new String(iidb); + } + + if (zooCache.get(Constants.ZROOT + "/" + instanceId) == null) { + if (instanceName == null) + throw new RuntimeException("Instance id " + instanceId + " does not exist in zookeeper"); + throw new RuntimeException("Instance id " + instanceId + " pointed to by the name " + instanceName + " does not exist in zookeeper"); + } + + return instanceId; + } + + @Override + public List getMasterLocations() { + if (closed) + throw new RuntimeException("ZooKeeperInstance has been closed."); + String masterLocPath = ZooUtil.getRoot(this) + Constants.ZMASTER_LOCK; + + OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up master location in zoocache."); + byte[] loc = ZooUtil.getLockData(zooCache, masterLocPath); + opTimer.stop("Found master at " + (loc == null ? null : new String(loc)) + " in %DURATION%"); + + if (loc == null) { + return Collections.emptyList(); + } + + return Collections.singletonList(new String(loc)); + } + + @Override + public String getRootTabletLocation() { + if (closed) + throw new RuntimeException("ZooKeeperInstance has been closed."); + String zRootLocPath = ZooUtil.getRoot(this) + Constants.ZROOT_TABLET_LOCATION; + + OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up root tablet location in zookeeper."); + byte[] loc = zooCache.get(zRootLocPath); + opTimer.stop("Found root tablet at " + (loc == null ? null : new String(loc)) + " in %DURATION%"); + + if (loc == null) { + return null; + } + + return new String(loc).split("\\|")[0]; + } + + @Override + public String getInstanceName() { + if (closed) + throw new RuntimeException("ZooKeeperInstance has been closed."); + if (instanceName == null) + instanceName = lookupInstanceName(zooCache, UUID.fromString(getInstanceID())); + + return instanceName; + } + + @Override + public String getZooKeepers() { + return zooKeepers; + } + + @Override + public int getZooKeepersSessionTimeOut() { + return zooKeepersSessionTimeOut; + } + + @Override + @Deprecated + public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException { + return getConnector(user, TextUtil.getBytes(new Text(pass.toString()))); + } + + @Override + @Deprecated + public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException { + return getConnector(user, ByteBufferUtil.toBytes(pass)); + } + + @Override + public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException { + return getConnector(CredentialHelper.create(principal, token, getInstanceID())); + } + + @SuppressWarnings("deprecation") + private Connector getConnector(TCredentials credential) throws AccumuloException, AccumuloSecurityException { + return new ConnectorImpl(this, credential); + } + + @Override + @Deprecated + public Connector getConnector(String principal, byte[] pass) throws AccumuloException, AccumuloSecurityException { + if (closed) { + throw new RuntimeException("ZooKeeperInstance has been closed."); + } else { + return getConnector(principal, new PasswordToken(pass)); + } + } + + private AccumuloConfiguration conf = null; + + @Override + public AccumuloConfiguration getConfiguration() { + if (conf == null) + conf = AccumuloConfiguration.getDefaultConfiguration(); + return conf; + } + + @Override + public void setConfiguration(AccumuloConfiguration conf) { + this.conf = conf; + } + + /** + * @deprecated Use {@link #lookupInstanceName(org.apache.accumulo.fate.zookeeper.ZooCache, UUID)} instead + */ + @Deprecated + public static String lookupInstanceName(org.apache.accumulo.core.zookeeper.ZooCache zooCache, UUID instanceId) { + return lookupInstanceName((ZooCache) zooCache, instanceId); + } + + /** + * Given a zooCache and instanceId, look up the instance name. + * + * @param zooCache + * @param instanceId + * @return the instance name + */ + public static String lookupInstanceName(ZooCache zooCache, UUID instanceId) { + ArgumentChecker.notNull(zooCache, instanceId); + for (String name : zooCache.getChildren(Constants.ZROOT + Constants.ZINSTANCES)) { + String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name; + UUID iid = UUID.fromString(new String(zooCache.get(instanceNamePath))); + if (iid.equals(instanceId)) { + return name; + } + } + return null; + } + + /** + * To be moved to server code. Only lives here to support certain client side utilities to minimize command-line options. + */ + @Deprecated + public static String getInstanceIDFromHdfs(Path instanceDirectory) { + try { + FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), AccumuloConfiguration.getSiteConfiguration()); + FileStatus[] files = null; + try { + files = fs.listStatus(instanceDirectory); + } catch (FileNotFoundException ex) { + // ignored + } + log.debug("Trying to read instance id from " + instanceDirectory); + if (files == null || files.length == 0) { + log.error("unable obtain instance id at " + instanceDirectory); + throw new RuntimeException("Accumulo not initialized, there is no instance id at " + instanceDirectory); + } else if (files.length != 1) { + log.error("multiple potential instances in " + instanceDirectory); + throw new RuntimeException("Accumulo found multiple possible instance ids in " + instanceDirectory); + } else { + String result = files[0].getPath().getName(); + return result; + } + } catch (IOException e) { + throw new RuntimeException("Accumulo not initialized, there is no instance id at " + instanceDirectory, e); + } + } + + @Deprecated + @Override + public Connector getConnector(org.apache.accumulo.core.security.thrift.AuthInfo auth) throws AccumuloException, AccumuloSecurityException { + return getConnector(auth.user, auth.password); + } + + static private final AtomicInteger clientInstances = new AtomicInteger(0); + + @Override - public synchronized void close() throws AccumuloException { ++ public synchronized void close() { + if (!closed && clientInstances.decrementAndGet() == 0) { + try { + zooCache.close(); + ThriftUtil.close(); - } catch (InterruptedException e) { ++ } catch (RuntimeException e) { + clientInstances.incrementAndGet(); - throw new AccumuloException("Issues closing ZooKeeper."); ++ throw e; + } + } + closed = true; + } + + @Override + public void finalize() { + // This method intentionally left blank. Users need to explicitly close Instances if they want things cleaned up nicely. + if (!closed) + log.warn("ZooKeeperInstance being cleaned up without being closed. Please remember to call close() before dereferencing to clean up threads."); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/7eb838e3/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java index 55213ef,0000000..d49c349 mode 100644,000000..100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java @@@ -1,171 -1,0 +1,171 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.client.mock; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.util.ByteBufferUtil; +import org.apache.accumulo.core.util.CachedConfiguration; +import org.apache.accumulo.core.util.TextUtil; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.Text; + +/** + * Mock Accumulo provides an in memory implementation of the Accumulo client API. It is possible that the behavior of this implementation may differ subtly from + * the behavior of Accumulo. This could result in unit tests that pass on Mock Accumulo and fail on Accumulo or visa-versa. Documenting the differences would be + * difficult and is not done. + * + *

+ * An alternative to Mock Accumulo called MiniAccumuloCluster was introduced in Accumulo 1.5. MiniAccumuloCluster spins up actual Accumulo server processes, can + * be used for unit testing, and its behavior should match Accumulo. The drawback of MiniAccumuloCluster is that it starts more slowly than Mock Accumulo. + * + */ + +public class MockInstance implements Instance { + + static final String genericAddress = "localhost:1234"; + static final Map instances = new HashMap(); + MockAccumulo acu; + String instanceName; + + public MockInstance() { + acu = new MockAccumulo(getDefaultFileSystem()); + instanceName = "mock-instance"; + } + + static FileSystem getDefaultFileSystem() { + try { + Configuration conf = CachedConfiguration.getInstance(); + conf.set("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem"); + conf.set("fs.default.name", "file:///"); + return FileSystem.get(CachedConfiguration.getInstance()); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + public MockInstance(String instanceName) { + this(instanceName, getDefaultFileSystem()); + } + + public MockInstance(String instanceName, FileSystem fs) { + synchronized (instances) { + if (instances.containsKey(instanceName)) + acu = instances.get(instanceName); + else + instances.put(instanceName, acu = new MockAccumulo(fs)); + } + this.instanceName = instanceName; + } + + @Override + public String getRootTabletLocation() { + return genericAddress; + } + + @Override + public List getMasterLocations() { + return Collections.singletonList(genericAddress); + } + + @Override + public String getInstanceID() { + return "mock-instance-id"; + } + + @Override + public String getInstanceName() { + return instanceName; + } + + @Override + public String getZooKeepers() { + return "localhost"; + } + + @Override + public int getZooKeepersSessionTimeOut() { + return 30 * 1000; + } + + @Override + @Deprecated + public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException { + return getConnector(user, new PasswordToken(pass)); + } + + @Override + @Deprecated + public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException { + return getConnector(user, ByteBufferUtil.toBytes(pass)); + } + + @Override + @Deprecated + public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException { + return getConnector(user, TextUtil.getBytes(new Text(pass.toString()))); + } + + AccumuloConfiguration conf = null; + + @Override + public AccumuloConfiguration getConfiguration() { + if (conf == null) + conf = AccumuloConfiguration.getDefaultConfiguration(); + return conf; + } + + @Override + public void setConfiguration(AccumuloConfiguration conf) { + this.conf = conf; + } + + @Deprecated + @Override + public Connector getConnector(org.apache.accumulo.core.security.thrift.AuthInfo auth) throws AccumuloException, AccumuloSecurityException { + return getConnector(auth.user, auth.password); + } + + @Override + public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException { + Connector conn = new MockConnector(principal, acu, this); + if (!acu.users.containsKey(principal)) + conn.securityOperations().createLocalUser(principal, (PasswordToken) token); + else if (!acu.users.get(principal).token.equals(token)) + throw new AccumuloSecurityException(principal, SecurityErrorCode.BAD_CREDENTIALS); + return conn; + } + + @Override - public void close() throws AccumuloException { ++ public void close() { + // NOOP + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/7eb838e3/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java index a42c280,0000000..c7fc707 mode 100644,000000..100644 --- a/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java @@@ -1,1285 -1,0 +1,1285 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.client.impl; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + +import junit.framework.TestCase; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocation; +import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocations; +import org.apache.accumulo.core.client.impl.TabletLocator.TabletServerMutations; +import org.apache.accumulo.core.client.impl.TabletLocatorImpl.TabletLocationObtainer; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.KeyExtent; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.PartialKey; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.thrift.TCredentials; +import org.apache.accumulo.core.util.MetadataTable; +import org.apache.accumulo.core.util.Pair; +import org.apache.hadoop.io.Text; + +public class TabletLocatorImplTest extends TestCase { + + private static final KeyExtent RTE = Constants.ROOT_TABLET_EXTENT; + private static final KeyExtent MTE = new KeyExtent(new Text(Constants.METADATA_TABLE_ID), null, RTE.getEndRow()); + private static TCredentials credential = null; + + static KeyExtent nke(String t, String er, String per) { + return new KeyExtent(new Text(t), er == null ? null : new Text(er), per == null ? null : new Text(per)); + } + + static Range nr(String k1, boolean si, String k2, boolean ei) { + return new Range(k1 == null ? null : new Text(k1), si, k2 == null ? null : new Text(k2), ei); + } + + static Range nr(String k1, String k2) { + return new Range(k1 == null ? null : new Text(k1), k2 == null ? null : new Text(k2)); + } + + static List nrl(Range... ranges) { + return Arrays.asList(ranges); + } + + static Object[] nol(Object... objs) { + return objs; + } + + @SuppressWarnings("unchecked") + static Map>> createExpectedBinnings(Object... data) { + + Map>> expBinnedRanges = new HashMap>>(); + + for (int i = 0; i < data.length; i += 2) { + String loc = (String) data[i]; + Object binData[] = (Object[]) data[i + 1]; + + HashMap> binnedKE = new HashMap>(); + + expBinnedRanges.put(loc, binnedKE); + + for (int j = 0; j < binData.length; j += 2) { + KeyExtent ke = (KeyExtent) binData[j]; + List ranges = (List) binData[j + 1]; + + binnedKE.put(ke, ranges); + } + } + + return expBinnedRanges; + } + + static TreeMap createMetaCacheKE(Object... data) { + TreeMap mcke = new TreeMap(); + + for (int i = 0; i < data.length; i += 2) { + KeyExtent ke = (KeyExtent) data[i]; + String loc = (String) data[i + 1]; + mcke.put(ke, new TabletLocation(ke, loc)); + } + + return mcke; + } + + static TreeMap createMetaCache(Object... data) { + TreeMap mcke = createMetaCacheKE(data); + + TreeMap mc = new TreeMap(TabletLocatorImpl.endRowComparator); + + for (Entry entry : mcke.entrySet()) { + if (entry.getKey().getEndRow() == null) + mc.put(TabletLocatorImpl.MAX_TEXT, entry.getValue()); + else + mc.put(entry.getKey().getEndRow(), entry.getValue()); + } + + return mc; + } + + static TabletLocatorImpl createLocators(TServers tservers, String rootTabLoc, String metaTabLoc, String table, Object... data) { + + TreeMap mcke = createMetaCacheKE(data); + + TestTabletLocationObtainer ttlo = new TestTabletLocationObtainer(tservers); + TestInstance testInstance = new TestInstance("instance1", "tserver1"); + + RootTabletLocator rtl = new RootTabletLocator(testInstance); + TabletLocatorImpl rootTabletCache = new TabletLocatorImpl(new Text(Constants.METADATA_TABLE_ID), rtl, ttlo); + TabletLocatorImpl tab1TabletCache = new TabletLocatorImpl(new Text(table), rootTabletCache, ttlo); + + setLocation(tservers, rootTabLoc, RTE, MTE, metaTabLoc); + + for (Entry entry : mcke.entrySet()) { + setLocation(tservers, metaTabLoc, MTE, entry.getKey(), entry.getValue().tablet_location); + } + + return tab1TabletCache; + + } + + static TabletLocatorImpl createLocators(String table, Object... data) { + TServers tservers = new TServers(); + return createLocators(tservers, "tserver1", "tserver2", table, data); + } + + private void runTest(Text tableName, List ranges, TabletLocatorImpl tab1TabletCache, Map>> expected) throws Exception { + List failures = Collections.emptyList(); + runTest(tableName, ranges, tab1TabletCache, expected, failures); + } + + private void runTest(Text tableName, List ranges, TabletLocatorImpl tab1TabletCache, Map>> expected, + List efailures) throws Exception { + + Map>> binnedRanges = new HashMap>>(); + List f = tab1TabletCache.binRanges(ranges, binnedRanges, credential); + assertEquals(expected, binnedRanges); + + HashSet f1 = new HashSet(f); + HashSet f2 = new HashSet(efailures); + + assertEquals(f2, f1); + } + + static Set nkes(KeyExtent... extents) { + HashSet kes = new HashSet(); + + for (KeyExtent keyExtent : extents) { + kes.add(keyExtent); + } + + return kes; + } + + static void runTest(TreeMap mc, KeyExtent remove, Set expected) { + // copy so same metaCache can be used for multiple test + + mc = new TreeMap(mc); + + TabletLocatorImpl.removeOverlapping(mc, remove); + + HashSet eic = new HashSet(); + for (TabletLocation tl : mc.values()) { + eic.add(tl.tablet_extent); + } + + assertEquals(expected, eic); + } + + static Mutation nm(String row, String... data) { + Mutation mut = new Mutation(new Text(row)); + + for (int i = 0; i < data.length; i++) { + String[] cvp = data[i].split("="); + String[] cols = cvp[0].split(":"); + + mut.put(new Text(cols[0]), new Text(cols[1]), new Value(cvp[1].getBytes())); + } + + return mut; + } + + static List nml(Mutation... ma) { + return Arrays.asList(ma); + } + + private void runTest(TabletLocatorImpl metaCache, List ml, Map>> emb, String... efailures) throws Exception { + Map binnedMutations = new HashMap(); + List afailures = new ArrayList(); + metaCache.binMutations(ml, binnedMutations, afailures, credential); + + verify(emb, binnedMutations); + + ArrayList afs = new ArrayList(); + ArrayList efs = new ArrayList(Arrays.asList(efailures)); + + for (Mutation mutation : afailures) { + afs.add(new String(mutation.getRow())); + } + + Collections.sort(afs); + Collections.sort(efs); + + assertEquals(efs, afs); + + } + + private void verify(Map>> expected, Map actual) { + assertEquals(expected.keySet(), actual.keySet()); + + for (String server : actual.keySet()) { + TabletServerMutations atb = actual.get(server); + Map> etb = expected.get(server); + + assertEquals(etb.keySet(), atb.getMutations().keySet()); + + for (KeyExtent ke : etb.keySet()) { + ArrayList eRows = new ArrayList(etb.get(ke)); + ArrayList aRows = new ArrayList(); + + for (Mutation m : atb.getMutations().get(ke)) { + aRows.add(new String(m.getRow())); + } + + Collections.sort(eRows); + Collections.sort(aRows); + + assertEquals(eRows, aRows); + } + } + + } + + static Map>> cemb(Object[]... ols) { + + Map>> emb = new HashMap>>(); + + for (Object[] ol : ols) { + String row = (String) ol[0]; + String server = (String) ol[1]; + KeyExtent ke = (KeyExtent) ol[2]; + + Map> tb = emb.get(server); + if (tb == null) { + tb = new HashMap>(); + emb.put(server, tb); + } + + List rl = tb.get(ke); + if (rl == null) { + rl = new ArrayList(); + tb.put(ke, rl); + } + + rl.add(row); + } + + return emb; + } + + public void testRemoveOverlapping1() { + TreeMap mc = createMetaCache(nke("0", null, null), "l1"); + + runTest(mc, nke("0", "a", null), nkes()); + runTest(mc, nke("0", null, null), nkes()); + runTest(mc, nke("0", null, "a"), nkes()); + + mc = createMetaCache(nke("0", "g", null), "l1", nke("0", "r", "g"), "l1", nke("0", null, "r"), "l1"); + runTest(mc, nke("0", null, null), nkes()); + + runTest(mc, nke("0", "a", null), nkes(nke("0", "r", "g"), nke("0", null, "r"))); + runTest(mc, nke("0", "g", null), nkes(nke("0", "r", "g"), nke("0", null, "r"))); + runTest(mc, nke("0", "h", null), nkes(nke("0", null, "r"))); + runTest(mc, nke("0", "r", null), nkes(nke("0", null, "r"))); + runTest(mc, nke("0", "s", null), nkes()); + + runTest(mc, nke("0", "b", "a"), nkes(nke("0", "r", "g"), nke("0", null, "r"))); + runTest(mc, nke("0", "g", "a"), nkes(nke("0", "r", "g"), nke("0", null, "r"))); + runTest(mc, nke("0", "h", "a"), nkes(nke("0", null, "r"))); + runTest(mc, nke("0", "r", "a"), nkes(nke("0", null, "r"))); + runTest(mc, nke("0", "s", "a"), nkes()); + + runTest(mc, nke("0", "h", "g"), nkes(nke("0", "g", null), nke("0", null, "r"))); + runTest(mc, nke("0", "r", "g"), nkes(nke("0", "g", null), nke("0", null, "r"))); + runTest(mc, nke("0", "s", "g"), nkes(nke("0", "g", null))); + + runTest(mc, nke("0", "i", "h"), nkes(nke("0", "g", null), nke("0", null, "r"))); + runTest(mc, nke("0", "r", "h"), nkes(nke("0", "g", null), nke("0", null, "r"))); + runTest(mc, nke("0", "s", "h"), nkes(nke("0", "g", null))); + + runTest(mc, nke("0", "z", "f"), nkes()); + runTest(mc, nke("0", "z", "g"), nkes(nke("0", "g", null))); + runTest(mc, nke("0", "z", "q"), nkes(nke("0", "g", null))); + runTest(mc, nke("0", "z", "r"), nkes(nke("0", "g", null), nke("0", "r", "g"))); + runTest(mc, nke("0", "z", "s"), nkes(nke("0", "g", null), nke("0", "r", "g"))); + + runTest(mc, nke("0", null, "f"), nkes()); + runTest(mc, nke("0", null, "g"), nkes(nke("0", "g", null))); + runTest(mc, nke("0", null, "q"), nkes(nke("0", "g", null))); + runTest(mc, nke("0", null, "r"), nkes(nke("0", "g", null), nke("0", "r", "g"))); + runTest(mc, nke("0", null, "s"), nkes(nke("0", "g", null), nke("0", "r", "g"))); + + } + + public void testRemoveOverlapping2() { + + // test removes when cache does not contain all tablets in a table + TreeMap mc = createMetaCache(nke("0", "r", "g"), "l1", nke("0", null, "r"), "l1"); + + runTest(mc, nke("0", "a", null), nkes(nke("0", "r", "g"), nke("0", null, "r"))); + runTest(mc, nke("0", "g", null), nkes(nke("0", "r", "g"), nke("0", null, "r"))); + runTest(mc, nke("0", "h", null), nkes(nke("0", null, "r"))); + runTest(mc, nke("0", "r", null), nkes(nke("0", null, "r"))); + runTest(mc, nke("0", "s", null), nkes()); + + runTest(mc, nke("0", "b", "a"), nkes(nke("0", "r", "g"), nke("0", null, "r"))); + runTest(mc, nke("0", "g", "a"), nkes(nke("0", "r", "g"), nke("0", null, "r"))); + runTest(mc, nke("0", "h", "a"), nkes(nke("0", null, "r"))); + runTest(mc, nke("0", "r", "a"), nkes(nke("0", null, "r"))); + runTest(mc, nke("0", "s", "a"), nkes()); + + runTest(mc, nke("0", "h", "g"), nkes(nke("0", null, "r"))); + + mc = createMetaCache(nke("0", "g", null), "l1", nke("0", null, "r"), "l1"); + + runTest(mc, nke("0", "h", "g"), nkes(nke("0", "g", null), nke("0", null, "r"))); + runTest(mc, nke("0", "h", "a"), nkes(nke("0", null, "r"))); + runTest(mc, nke("0", "s", "g"), nkes(nke("0", "g", null))); + runTest(mc, nke("0", "s", "a"), nkes()); + + mc = createMetaCache(nke("0", "g", null), "l1", nke("0", "r", "g"), "l1"); + + runTest(mc, nke("0", "z", "f"), nkes()); + runTest(mc, nke("0", "z", "g"), nkes(nke("0", "g", null))); + runTest(mc, nke("0", "z", "q"), nkes(nke("0", "g", null))); + runTest(mc, nke("0", "z", "r"), nkes(nke("0", "g", null), nke("0", "r", "g"))); + runTest(mc, nke("0", "z", "s"), nkes(nke("0", "g", null), nke("0", "r", "g"))); + + runTest(mc, nke("0", null, "f"), nkes()); + runTest(mc, nke("0", null, "g"), nkes(nke("0", "g", null))); + runTest(mc, nke("0", null, "q"), nkes(nke("0", "g", null))); + runTest(mc, nke("0", null, "r"), nkes(nke("0", "g", null), nke("0", "r", "g"))); + runTest(mc, nke("0", null, "s"), nkes(nke("0", "g", null), nke("0", "r", "g"))); + } + + static class TestInstance implements Instance { + + private final String iid; + private String rtl; + + public TestInstance(String iid, String rtl) { + this.iid = iid; + this.rtl = rtl; + } + + @Override + public String getInstanceID() { + return iid; + } + + @Override + public String getInstanceName() { + throw new UnsupportedOperationException(); + } + + @Override + public List getMasterLocations() { + throw new UnsupportedOperationException(); + } + + @Override + public String getRootTabletLocation() { + return rtl; + } + + @Override + public String getZooKeepers() { + throw new UnsupportedOperationException(); + } + + @Override + public int getZooKeepersSessionTimeOut() { + throw new UnsupportedOperationException(); + } + + public void setRootTabletLocation(String rtl) { + this.rtl = rtl; + } + + @Override + @Deprecated + public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException { + throw new UnsupportedOperationException(); + } + + @Override + @Deprecated + public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException { + throw new UnsupportedOperationException(); + } + + AccumuloConfiguration conf = AccumuloConfiguration.getDefaultConfiguration(); + + @Override + public AccumuloConfiguration getConfiguration() { + return conf; + } + + @Override + public void setConfiguration(AccumuloConfiguration conf) { + this.conf = conf; + } + + @Override + @Deprecated + public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public Connector getConnector(org.apache.accumulo.core.security.thrift.AuthInfo auth) throws AccumuloException, AccumuloSecurityException { + return getConnector(auth.user, auth.getPassword()); + } + + @Override + public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException { + throw new UnsupportedOperationException(); + } + + @Override - public void close() throws AccumuloException { ++ public void close() { + // NOOP + } + } + + static class TServers { + private final Map>> tservers = new HashMap>>(); + } + + static class TestTabletLocationObtainer implements TabletLocationObtainer { + + private final Map>> tservers; + + TestTabletLocationObtainer(TServers tservers) { + this.tservers = tservers.tservers; + } + + @Override + public TabletLocations lookupTablet(TabletLocation src, Text row, Text stopRow, TabletLocator parent, TCredentials credentials) throws AccumuloSecurityException { + + // System.out.println("lookupTablet("+src+","+row+","+stopRow+","+ parent+")"); + // System.out.println(tservers); + + ArrayList list = new ArrayList(); + + Map> tablets = tservers.get(src.tablet_location); + + if (tablets == null) { + parent.invalidateCache(src.tablet_location); + return null; + } + + SortedMap tabletData = tablets.get(src.tablet_extent); + + if (tabletData == null) { + parent.invalidateCache(src.tablet_extent); + return null; + } + + // the following clip is done on a tablet, do it here to see if it throws exceptions + src.tablet_extent.toDataRange().clip(new Range(row, true, stopRow, true)); + + Key startKey = new Key(row); + Key stopKey = new Key(stopRow).followingKey(PartialKey.ROW); + + SortedMap results = tabletData.tailMap(startKey).headMap(stopKey); + + Pair,List> metadata = MetadataTable.getMetadataLocationEntries(results); + + for (Entry entry : metadata.getFirst().entrySet()) { + list.add(new TabletLocation(entry.getKey(), entry.getValue().toString())); + } + + return new TabletLocations(list, metadata.getSecond()); + } + + @Override + public List lookupTablets(String tserver, Map> map, TabletLocator parent, TCredentials credentials) + throws AccumuloSecurityException { + + ArrayList list = new ArrayList(); + + Map> tablets = tservers.get(tserver); + + if (tablets == null) { + parent.invalidateCache(tserver); + return list; + } + + TreeMap results = new TreeMap(); + + Set>> es = map.entrySet(); + List failures = new ArrayList(); + for (Entry> entry : es) { + SortedMap tabletData = tablets.get(entry.getKey()); + + if (tabletData == null) { + failures.add(entry.getKey()); + continue; + } + List ranges = entry.getValue(); + for (Range range : ranges) { + SortedMap tm; + if (range.getStartKey() == null) + tm = tabletData; + else + tm = tabletData.tailMap(range.getStartKey()); + + for (Entry de : tm.entrySet()) { + if (range.afterEndKey(de.getKey())) { + break; + } + + if (range.contains(de.getKey())) { + results.put(de.getKey(), de.getValue()); + } + } + } + } + + if (failures.size() > 0) + parent.invalidateCache(failures); + + SortedMap metadata = MetadataTable.getMetadataLocationEntries(results).getFirst(); + + for (Entry entry : metadata.entrySet()) { + list.add(new TabletLocation(entry.getKey(), entry.getValue().toString())); + } + + return list; + + } + + } + + static void createEmptyTablet(TServers tservers, String server, KeyExtent tablet) { + Map> tablets = tservers.tservers.get(server); + if (tablets == null) { + tablets = new HashMap>(); + tservers.tservers.put(server, tablets); + } + + SortedMap tabletData = tablets.get(tablet); + if (tabletData == null) { + tabletData = new TreeMap(); + tablets.put(tablet, tabletData); + } else if (tabletData.size() > 0) { + throw new RuntimeException("Asked for empty tablet, but non empty tablet exists"); + } + } + + static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyExtent ke, String location, String instance) { + Map> tablets = tservers.tservers.get(server); + if (tablets == null) { + tablets = new HashMap>(); + tservers.tservers.put(server, tablets); + } + + SortedMap tabletData = tablets.get(tablet); + if (tabletData == null) { + tabletData = new TreeMap(); + tablets.put(tablet, tabletData); + } + + Text mr = ke.getMetadataEntry(); + Value per = KeyExtent.encodePrevEndRow(ke.getPrevEndRow()); + + if (location != null) { + if (instance == null) + instance = ""; + Key lk = new Key(mr, Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, new Text(instance)); + tabletData.put(lk, new Value(location.getBytes())); + } + + Key pk = new Key(mr, Constants.METADATA_PREV_ROW_COLUMN.getColumnFamily(), Constants.METADATA_PREV_ROW_COLUMN.getColumnQualifier()); + tabletData.put(pk, per); + } + + static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyExtent ke, String location) { + setLocation(tservers, server, tablet, ke, location, ""); + } + + static void deleteServer(TServers tservers, String server) { + tservers.tservers.remove(server); + + } + + private void locateTabletTest(TabletLocatorImpl cache, String row, boolean skipRow, KeyExtent expected, String server, TCredentials credentials) + throws Exception { + TabletLocation tl = cache.locateTablet(new Text(row), skipRow, false, credentials); + + if (expected == null) { + if (tl != null) + System.out.println("tl = " + tl); + assertNull(tl); + } else { + assertNotNull(tl); + assertEquals(server, tl.tablet_location); + assertEquals(expected, tl.tablet_extent); + } + } + + private void locateTabletTest(TabletLocatorImpl cache, String row, KeyExtent expected, String server, TCredentials credentials) throws Exception { + locateTabletTest(cache, row, false, expected, server, credentials); + } + + public void test1() throws Exception { + TServers tservers = new TServers(); + TestTabletLocationObtainer ttlo = new TestTabletLocationObtainer(tservers); + TestInstance testInstance = new TestInstance("instance1", "tserver1"); + + RootTabletLocator rtl = new RootTabletLocator(testInstance); + TabletLocatorImpl rootTabletCache = new TabletLocatorImpl(new Text(Constants.METADATA_TABLE_ID), rtl, ttlo); + TabletLocatorImpl tab1TabletCache = new TabletLocatorImpl(new Text("tab1"), rootTabletCache, ttlo); + + locateTabletTest(tab1TabletCache, "r1", null, null, credential); + + KeyExtent tab1e = nke("tab1", null, null); + + setLocation(tservers, "tserver1", RTE, MTE, "tserver2"); + setLocation(tservers, "tserver2", MTE, tab1e, "tserver3"); + + locateTabletTest(tab1TabletCache, "r1", tab1e, "tserver3", credential); + locateTabletTest(tab1TabletCache, "r2", tab1e, "tserver3", credential); + + // simulate a split + KeyExtent tab1e1 = nke("tab1", "g", null); + KeyExtent tab1e2 = nke("tab1", null, "g"); + + setLocation(tservers, "tserver2", MTE, tab1e1, "tserver4"); + setLocation(tservers, "tserver2", MTE, tab1e2, "tserver5"); + + locateTabletTest(tab1TabletCache, "r1", tab1e, "tserver3", credential); + tab1TabletCache.invalidateCache(tab1e); + locateTabletTest(tab1TabletCache, "r1", tab1e2, "tserver5", credential); + locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver4", credential); + locateTabletTest(tab1TabletCache, "a", true, tab1e1, "tserver4", credential); + locateTabletTest(tab1TabletCache, "g", tab1e1, "tserver4", credential); + locateTabletTest(tab1TabletCache, "g", true, tab1e2, "tserver5", credential); + + // simulate a partial split + KeyExtent tab1e22 = nke("tab1", null, "m"); + setLocation(tservers, "tserver2", MTE, tab1e22, "tserver6"); + locateTabletTest(tab1TabletCache, "r1", tab1e2, "tserver5", credential); + tab1TabletCache.invalidateCache(tab1e2); + locateTabletTest(tab1TabletCache, "r1", tab1e22, "tserver6", credential); + locateTabletTest(tab1TabletCache, "h", null, null, credential); + locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver4", credential); + KeyExtent tab1e21 = nke("tab1", "m", "g"); + setLocation(tservers, "tserver2", MTE, tab1e21, "tserver7"); + locateTabletTest(tab1TabletCache, "r1", tab1e22, "tserver6", credential); + locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver7", credential); + locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver4", credential); + + // simulate a migration + setLocation(tservers, "tserver2", MTE, tab1e21, "tserver8"); + tab1TabletCache.invalidateCache(tab1e21); + locateTabletTest(tab1TabletCache, "r1", tab1e22, "tserver6", credential); + locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver8", credential); + locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver4", credential); + + // simulate a server failure + setLocation(tservers, "tserver2", MTE, tab1e21, "tserver9"); + tab1TabletCache.invalidateCache("tserver8"); + locateTabletTest(tab1TabletCache, "r1", tab1e22, "tserver6", credential); + locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver9", credential); + locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver4", credential); + + // simulate all servers failing + deleteServer(tservers, "tserver1"); + deleteServer(tservers, "tserver2"); + tab1TabletCache.invalidateCache("tserver4"); + tab1TabletCache.invalidateCache("tserver6"); + tab1TabletCache.invalidateCache("tserver9"); + + locateTabletTest(tab1TabletCache, "r1", null, null, credential); + locateTabletTest(tab1TabletCache, "h", null, null, credential); + locateTabletTest(tab1TabletCache, "a", null, null, credential); + + testInstance.setRootTabletLocation("tserver4"); + setLocation(tservers, "tserver4", RTE, MTE, "tserver5"); + setLocation(tservers, "tserver5", MTE, tab1e1, "tserver1"); + setLocation(tservers, "tserver5", MTE, tab1e21, "tserver2"); + setLocation(tservers, "tserver5", MTE, tab1e22, "tserver3"); + + locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver1", credential); + locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver2", credential); + locateTabletTest(tab1TabletCache, "r", tab1e22, "tserver3", credential); + + // simulate the !METADATA table splitting + KeyExtent mte1 = new KeyExtent(new Text(Constants.METADATA_TABLE_ID), tab1e21.getMetadataEntry(), RTE.getEndRow()); + KeyExtent mte2 = new KeyExtent(new Text(Constants.METADATA_TABLE_ID), null, tab1e21.getMetadataEntry()); + + setLocation(tservers, "tserver4", RTE, mte1, "tserver5"); + setLocation(tservers, "tserver4", RTE, mte2, "tserver6"); + deleteServer(tservers, "tserver5"); + setLocation(tservers, "tserver5", mte1, tab1e1, "tserver7"); + setLocation(tservers, "tserver5", mte1, tab1e21, "tserver8"); + setLocation(tservers, "tserver6", mte2, tab1e22, "tserver9"); + + tab1TabletCache.invalidateCache(tab1e1); + tab1TabletCache.invalidateCache(tab1e21); + tab1TabletCache.invalidateCache(tab1e22); + + locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver7", credential); + locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver8", credential); + locateTabletTest(tab1TabletCache, "r", tab1e22, "tserver9", credential); + + // simulate metadata and regular server down and the reassigned + deleteServer(tservers, "tserver5"); + tab1TabletCache.invalidateCache("tserver7"); + locateTabletTest(tab1TabletCache, "a", null, null, credential); + locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver8", credential); + locateTabletTest(tab1TabletCache, "r", tab1e22, "tserver9", credential); + + setLocation(tservers, "tserver4", RTE, mte1, "tserver10"); + setLocation(tservers, "tserver10", mte1, tab1e1, "tserver7"); + setLocation(tservers, "tserver10", mte1, tab1e21, "tserver8"); + + locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver7", credential); + locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver8", credential); + locateTabletTest(tab1TabletCache, "r", tab1e22, "tserver9", credential); + tab1TabletCache.invalidateCache("tserver7"); + setLocation(tservers, "tserver10", mte1, tab1e1, "tserver2"); + locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver2", credential); + locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver8", credential); + locateTabletTest(tab1TabletCache, "r", tab1e22, "tserver9", credential); + + // simulate a hole in the !METADATA table, caused by a partial split + KeyExtent mte11 = new KeyExtent(new Text(Constants.METADATA_TABLE_ID), tab1e1.getMetadataEntry(), RTE.getEndRow()); + KeyExtent mte12 = new KeyExtent(new Text(Constants.METADATA_TABLE_ID), tab1e21.getMetadataEntry(), tab1e1.getMetadataEntry()); + deleteServer(tservers, "tserver10"); + setLocation(tservers, "tserver4", RTE, mte12, "tserver10"); + setLocation(tservers, "tserver10", mte12, tab1e21, "tserver12"); + + // at this point should be no info in !METADATA about tab1e1 + tab1TabletCache.invalidateCache(tab1e1); + tab1TabletCache.invalidateCache(tab1e21); + locateTabletTest(tab1TabletCache, "a", null, null, credential); + locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver12", credential); + locateTabletTest(tab1TabletCache, "r", tab1e22, "tserver9", credential); + + setLocation(tservers, "tserver4", RTE, mte11, "tserver5"); + setLocation(tservers, "tserver5", mte11, tab1e1, "tserver13"); + + locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver13", credential); + locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver12", credential); + locateTabletTest(tab1TabletCache, "r", tab1e22, "tserver9", credential); + } + + public void test2() throws Exception { + TServers tservers = new TServers(); + TabletLocatorImpl metaCache = createLocators(tservers, "tserver1", "tserver2", "foo"); + + KeyExtent ke1 = nke("foo", "m", null); + KeyExtent ke2 = nke("foo", null, "m"); + + setLocation(tservers, "tserver2", MTE, ke1, null); + setLocation(tservers, "tserver2", MTE, ke2, "L1"); + + locateTabletTest(metaCache, "a", null, null, credential); + locateTabletTest(metaCache, "r", ke2, "L1", credential); + + setLocation(tservers, "tserver2", MTE, ke1, "L2"); + + locateTabletTest(metaCache, "a", ke1, "L2", credential); + locateTabletTest(metaCache, "r", ke2, "L1", credential); + } + + public void testBinRanges1() throws Exception { + Text tableName = new Text("foo"); + + TabletLocatorImpl metaCache = createLocators("foo", nke("foo", null, null), "l1"); + + List ranges = nrl(nr(null, null)); + Map>> expected = createExpectedBinnings("l1", nol(nke("foo", null, null), nrl(nr(null, null))) + + ); + + runTest(tableName, ranges, metaCache, expected); + + ranges = nrl(nr("a", null)); + expected = createExpectedBinnings("l1", nol(nke("foo", null, null), nrl(nr("a", null))) + + ); + + runTest(tableName, ranges, metaCache, expected); + + ranges = nrl(nr(null, "b")); + expected = createExpectedBinnings("l1", nol(nke("foo", null, null), nrl(nr(null, "b"))) + + ); + + runTest(tableName, ranges, metaCache, expected); + } + + public void testBinRanges2() throws Exception { + + Text tableName = new Text("foo"); + + List ranges = nrl(nr(null, null)); + TabletLocatorImpl metaCache = createLocators("foo", nke("foo", "g", null), "l1", nke("foo", null, "g"), "l2"); + + Map>> expected = createExpectedBinnings("l1", nol(nke("foo", "g", null), nrl(nr(null, null))), "l2", + nol(nke("foo", null, "g"), nrl(nr(null, null))) + + ); + + runTest(tableName, ranges, metaCache, expected); + } + + public void testBinRanges3() throws Exception { + + Text tableName = new Text("foo"); + + // test with three tablets and a range that covers the whole table + List ranges = nrl(nr(null, null)); + TabletLocatorImpl metaCache = createLocators("foo", nke("foo", "g", null), "l1", nke("foo", "m", "g"), "l2", nke("foo", null, "m"), "l2"); + + Map>> expected = createExpectedBinnings("l1", nol(nke("foo", "g", null), nrl(nr(null, null))), "l2", + nol(nke("foo", "m", "g"), nrl(nr(null, null)), nke("foo", null, "m"), nrl(nr(null, null))) + + ); + + runTest(tableName, ranges, metaCache, expected); + + // test with three tablets where one range falls within the first tablet and last two ranges fall within the last tablet + ranges = nrl(nr(null, "c"), nr("s", "y"), nr("z", null)); + expected = createExpectedBinnings("l1", nol(nke("foo", "g", null), nrl(nr(null, "c"))), "l2", nol(nke("foo", null, "m"), nrl(nr("s", "y"), nr("z", null))) + + ); + + runTest(tableName, ranges, metaCache, expected); + + // test is same as above, but has an additional range that spans the first two tablets + ranges = nrl(nr(null, "c"), nr("f", "i"), nr("s", "y"), nr("z", null)); + expected = createExpectedBinnings("l1", nol(nke("foo", "g", null), nrl(nr(null, "c"), nr("f", "i"))), "l2", + nol(nke("foo", "m", "g"), nrl(nr("f", "i")), nke("foo", null, "m"), nrl(nr("s", "y"), nr("z", null))) + + ); + + runTest(tableName, ranges, metaCache, expected); + + // test where start of range is not inclusive and same as tablet endrow + ranges = nrl(nr("g", false, "m", true)); + expected = createExpectedBinnings("l2", nol(nke("foo", "m", "g"), nrl(nr("g", false, "m", true))) + + ); + + runTest(tableName, ranges, metaCache, expected); + + // test where start of range is inclusive and same as tablet endrow + ranges = nrl(nr("g", true, "m", true)); + expected = createExpectedBinnings("l1", nol(nke("foo", "g", null), nrl(nr("g", true, "m", true))), "l2", + nol(nke("foo", "m", "g"), nrl(nr("g", true, "m", true))) + + ); + + runTest(tableName, ranges, metaCache, expected); + + ranges = nrl(nr("g", true, "m", false)); + expected = createExpectedBinnings("l1", nol(nke("foo", "g", null), nrl(nr("g", true, "m", false))), "l2", + nol(nke("foo", "m", "g"), nrl(nr("g", true, "m", false))) + + ); + + runTest(tableName, ranges, metaCache, expected); + + ranges = nrl(nr("g", false, "m", false)); + expected = createExpectedBinnings("l2", nol(nke("foo", "m", "g"), nrl(nr("g", false, "m", false))) + + ); + + runTest(tableName, ranges, metaCache, expected); + } + + public void testBinRanges4() throws Exception { + Text tableName = new Text("foo"); + + List ranges = nrl(new Range(new Text("1"))); + TabletLocatorImpl metaCache = createLocators("foo", nke("foo", "0", null), "l1", nke("foo", "1", "0"), "l2", nke("foo", "2", "1"), "l3", + nke("foo", "3", "2"), "l4", nke("foo", null, "3"), "l5"); + + Map>> expected = createExpectedBinnings("l2", nol(nke("foo", "1", "0"), nrl(new Range(new Text("1")))) + + ); + + runTest(tableName, ranges, metaCache, expected); + + Key rowColKey = new Key(new Text("3"), new Text("cf1"), new Text("cq1")); + Range range = new Range(rowColKey, true, new Key(new Text("3")).followingKey(PartialKey.ROW), false); + + ranges = nrl(range); + Map>> expected4 = createExpectedBinnings("l4", nol(nke("foo", "3", "2"), nrl(range)) + + ); + + runTest(tableName, ranges, metaCache, expected4, nrl()); + + range = new Range(rowColKey, true, new Key(new Text("3")).followingKey(PartialKey.ROW), true); + + ranges = nrl(range); + Map>> expected5 = createExpectedBinnings("l4", nol(nke("foo", "3", "2"), nrl(range)), "l5", + nol(nke("foo", null, "3"), nrl(range)) + + ); + + runTest(tableName, ranges, metaCache, expected5, nrl()); + + range = new Range(new Text("2"), false, new Text("3"), false); + ranges = nrl(range); + Map>> expected6 = createExpectedBinnings("l4", nol(nke("foo", "3", "2"), nrl(range)) + + ); + runTest(tableName, ranges, metaCache, expected6, nrl()); + + range = new Range(new Text("2"), true, new Text("3"), false); + ranges = nrl(range); + Map>> expected7 = createExpectedBinnings("l3", nol(nke("foo", "2", "1"), nrl(range)), "l4", + nol(nke("foo", "3", "2"), nrl(range)) + + ); + runTest(tableName, ranges, metaCache, expected7, nrl()); + + range = new Range(new Text("2"), false, new Text("3"), true); + ranges = nrl(range); + Map>> expected8 = createExpectedBinnings("l4", nol(nke("foo", "3", "2"), nrl(range)) + + ); + runTest(tableName, ranges, metaCache, expected8, nrl()); + + range = new Range(new Text("2"), true, new Text("3"), true); + ranges = nrl(range); + Map>> expected9 = createExpectedBinnings("l3", nol(nke("foo", "2", "1"), nrl(range)), "l4", + nol(nke("foo", "3", "2"), nrl(range)) + + ); + runTest(tableName, ranges, metaCache, expected9, nrl()); + + } + + public void testBinRanges5() throws Exception { + // Test binning when there is a hole in the !METADATA information + Text tableName = new Text("foo"); + + List ranges = nrl(new Range(new Text("1"))); + TabletLocatorImpl metaCache = createLocators("foo", nke("foo", "0", null), "l1", nke("foo", "1", "0"), "l2", nke("foo", "3", "2"), "l4", + nke("foo", null, "3"), "l5"); + + Map>> expected1 = createExpectedBinnings("l2", nol(nke("foo", "1", "0"), nrl(new Range(new Text("1")))) + + ); + + runTest(tableName, ranges, metaCache, expected1); + + ranges = nrl(new Range(new Text("2")), new Range(new Text("11"))); + Map>> expected2 = createExpectedBinnings(); + + runTest(tableName, ranges, metaCache, expected2, ranges); + + ranges = nrl(new Range(new Text("1")), new Range(new Text("2"))); + + runTest(tableName, ranges, metaCache, expected1, nrl(new Range(new Text("2")))); + + ranges = nrl(nr("0", "2"), nr("3", "4")); + Map>> expected3 = createExpectedBinnings("l4", nol(nke("foo", "3", "2"), nrl(nr("3", "4"))), "l5", + nol(nke("foo", null, "3"), nrl(nr("3", "4"))) + + ); + + runTest(tableName, ranges, metaCache, expected3, nrl(nr("0", "2"))); + + ranges = nrl(nr("0", "1"), nr("0", "11"), nr("1", "2"), nr("0", "4"), nr("2", "4"), nr("21", "4")); + Map>> expected4 = createExpectedBinnings("l1", nol(nke("foo", "0", null), nrl(nr("0", "1"))), "l2", + nol(nke("foo", "1", "0"), nrl(nr("0", "1"))), "l4", nol(nke("foo", "3", "2"), nrl(nr("21", "4"))), "l5", nol(nke("foo", null, "3"), nrl(nr("21", "4"))) + + ); + + runTest(tableName, ranges, metaCache, expected4, nrl(nr("0", "11"), nr("1", "2"), nr("0", "4"), nr("2", "4"))); + } + + public void testBinMutations1() throws Exception { + // one tablet table + KeyExtent ke1 = nke("foo", null, null); + TabletLocatorImpl metaCache = createLocators("foo", ke1, "l1"); + + List ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2"), nm("c", "cf1:cq1=v3", "cf1:cq2=v4")); + Map>> emb = cemb(nol("a", "l1", ke1), nol("c", "l1", ke1)); + runTest(metaCache, ml, emb); + + ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2")); + emb = cemb(nol("a", "l1", ke1)); + runTest(metaCache, ml, emb); + + ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2"), nm("a", "cf1:cq3=v3")); + emb = cemb(nol("a", "l1", ke1), nol("a", "l1", ke1)); + runTest(metaCache, ml, emb); + + } + + public void testBinMutations2() throws Exception { + // no tablets for table + TabletLocatorImpl metaCache = createLocators("foo"); + + List ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2"), nm("c", "cf1:cq1=v3", "cf1:cq2=v4")); + Map>> emb = cemb(); + runTest(metaCache, ml, emb, "a", "c"); + } + + public void testBinMutations3() throws Exception { + // three tablet table + KeyExtent ke1 = nke("foo", "h", null); + KeyExtent ke2 = nke("foo", "t", "h"); + KeyExtent ke3 = nke("foo", null, "t"); + + TabletLocatorImpl metaCache = createLocators("foo", ke1, "l1", ke2, "l2", ke3, "l3"); + + List ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2"), nm("i", "cf1:cq1=v3", "cf1:cq2=v4")); + Map>> emb = cemb(nol("a", "l1", ke1), nol("i", "l2", ke2)); + runTest(metaCache, ml, emb); + + ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2")); + emb = cemb(nol("a", "l1", ke1)); + runTest(metaCache, ml, emb); + + ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2"), nm("a", "cf1:cq3=v3")); + emb = cemb(nol("a", "l1", ke1), nol("a", "l1", ke1)); + runTest(metaCache, ml, emb); + + ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2"), nm("w", "cf1:cq3=v3")); + emb = cemb(nol("a", "l1", ke1), nol("w", "l3", ke3)); + runTest(metaCache, ml, emb); + + ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2"), nm("w", "cf1:cq3=v3"), nm("z", "cf1:cq4=v4")); + emb = cemb(nol("a", "l1", ke1), nol("w", "l3", ke3), nol("z", "l3", ke3)); + runTest(metaCache, ml, emb); + + ml = nml(nm("h", "cf1:cq1=v1", "cf1:cq2=v2"), nm("t", "cf1:cq1=v1", "cf1:cq2=v2")); + emb = cemb(nol("h", "l1", ke1), nol("t", "l2", ke2)); + runTest(metaCache, ml, emb); + } + + public void testBinMutations4() throws Exception { + // three table with hole + KeyExtent ke1 = nke("foo", "h", null); + + KeyExtent ke3 = nke("foo", null, "t"); + + TabletLocatorImpl metaCache = createLocators("foo", ke1, "l1", ke3, "l3"); + + List ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2"), nm("i", "cf1:cq1=v3", "cf1:cq2=v4")); + Map>> emb = cemb(nol("a", "l1", ke1)); + runTest(metaCache, ml, emb, "i"); + + ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2")); + emb = cemb(nol("a", "l1", ke1)); + runTest(metaCache, ml, emb); + + ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2"), nm("a", "cf1:cq3=v3")); + emb = cemb(nol("a", "l1", ke1), nol("a", "l1", ke1)); + runTest(metaCache, ml, emb); + + ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2"), nm("w", "cf1:cq3=v3")); + emb = cemb(nol("a", "l1", ke1), nol("w", "l3", ke3)); + runTest(metaCache, ml, emb); + + ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2"), nm("w", "cf1:cq3=v3"), nm("z", "cf1:cq4=v4")); + emb = cemb(nol("a", "l1", ke1), nol("w", "l3", ke3), nol("z", "l3", ke3)); + runTest(metaCache, ml, emb); + + ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2"), nm("w", "cf1:cq3=v3"), nm("z", "cf1:cq4=v4"), nm("t", "cf1:cq5=v5")); + emb = cemb(nol("a", "l1", ke1), nol("w", "l3", ke3), nol("z", "l3", ke3)); + runTest(metaCache, ml, emb, "t"); + } + + public void testBinSplit() throws Exception { + // try binning mutations and ranges when a tablet splits + + for (int i = 0; i < 3; i++) { + // when i == 0 only test binning mutations + // when i == 1 only test binning ranges + // when i == 2 test both + + KeyExtent ke1 = nke("foo", null, null); + TServers tservers = new TServers(); + TabletLocatorImpl metaCache = createLocators(tservers, "tserver1", "tserver2", "foo", ke1, "l1"); + + List ml = nml(nm("a", "cf1:cq1=v1", "cf1:cq2=v2"), nm("m", "cf1:cq1=v3", "cf1:cq2=v4"), nm("z", "cf1:cq1=v5")); + Map>> emb = cemb(nol("a", "l1", ke1), nol("m", "l1", ke1), nol("z", "l1", ke1)); + if (i == 0 || i == 2) + runTest(metaCache, ml, emb); + + List ranges = nrl(new Range(new Text("a")), new Range(new Text("m")), new Range(new Text("z"))); + + Map>> expected1 = createExpectedBinnings("l1", nol(nke("foo", null, null), ranges) + + ); + + if (i == 1 || i == 2) + runTest(new Text("foo"), ranges, metaCache, expected1); + + KeyExtent ke11 = nke("foo", "n", null); + KeyExtent ke12 = nke("foo", null, "n"); + + setLocation(tservers, "tserver2", MTE, ke12, "l2"); + + metaCache.invalidateCache(ke1); + + emb = cemb(nol("z", "l2", ke12)); + if (i == 0 || i == 2) + runTest(metaCache, ml, emb, "a", "m"); + + Map>> expected2 = createExpectedBinnings("l2", nol(nke("foo", null, "n"), nrl(new Range(new Text("z")))) + + ); + + if (i == 1 || i == 2) + runTest(new Text("foo"), ranges, metaCache, expected2, nrl(new Range(new Text("a")), new Range(new Text("m")))); + + setLocation(tservers, "tserver2", MTE, ke11, "l3"); + emb = cemb(nol("a", "l3", ke11), nol("m", "l3", ke11), nol("z", "l2", ke12)); + if (i == 0 || i == 2) + runTest(metaCache, ml, emb); + + Map>> expected3 = createExpectedBinnings("l2", nol(nke("foo", null, "n"), nrl(new Range(new Text("z")))), "l3", + nol(nke("foo", "n", null), nrl(new Range(new Text("a")), new Range(new Text("m")))) + + ); + + if (i == 1 || i == 2) + runTest(new Text("foo"), ranges, metaCache, expected3); + } + } + + public void testBug1() throws Exception { + // a bug that occurred while running continuous ingest + KeyExtent mte1 = new KeyExtent(new Text(Constants.METADATA_TABLE_ID), new Text("0;0bc"), RTE.getEndRow()); + KeyExtent mte2 = new KeyExtent(new Text(Constants.METADATA_TABLE_ID), null, new Text("0;0bc")); + + TServers tservers = new TServers(); + TestTabletLocationObtainer ttlo = new TestTabletLocationObtainer(tservers); + TestInstance testInstance = new TestInstance("instance1", "tserver1"); + + RootTabletLocator rtl = new RootTabletLocator(testInstance); + TabletLocatorImpl rootTabletCache = new TabletLocatorImpl(new Text(Constants.METADATA_TABLE_ID), rtl, ttlo); + TabletLocatorImpl tab0TabletCache = new TabletLocatorImpl(new Text("0"), rootTabletCache, ttlo); + + setLocation(tservers, "tserver1", RTE, mte1, "tserver2"); + setLocation(tservers, "tserver1", RTE, mte2, "tserver3"); + + // create two tablets that straddle a !METADATA split point + KeyExtent ke1 = new KeyExtent(new Text("0"), new Text("0bbf20e"), null); + KeyExtent ke2 = new KeyExtent(new Text("0"), new Text("0bc0756"), new Text("0bbf20e")); + + setLocation(tservers, "tserver2", mte1, ke1, "tserver4"); + setLocation(tservers, "tserver3", mte2, ke2, "tserver5"); + + // look up something that comes after the last entry in mte1 + locateTabletTest(tab0TabletCache, "0bbff", ke2, "tserver5", credential); + } + + public void testBug2() throws Exception { + // a bug that occurred while running a functional test + KeyExtent mte1 = new KeyExtent(new Text(Constants.METADATA_TABLE_ID), new Text("~"), RTE.getEndRow()); + KeyExtent mte2 = new KeyExtent(new Text(Constants.METADATA_TABLE_ID), null, new Text("~")); + + TServers tservers = new TServers(); + TestTabletLocationObtainer ttlo = new TestTabletLocationObtainer(tservers); + TestInstance testInstance = new TestInstance("instance1", "tserver1"); + + RootTabletLocator rtl = new RootTabletLocator(testInstance); + TabletLocatorImpl rootTabletCache = new TabletLocatorImpl(new Text(Constants.METADATA_TABLE_ID), rtl, ttlo); + TabletLocatorImpl tab0TabletCache = new TabletLocatorImpl(new Text("0"), rootTabletCache, ttlo); + + setLocation(tservers, "tserver1", RTE, mte1, "tserver2"); + setLocation(tservers, "tserver1", RTE, mte2, "tserver3"); + + // create the ~ tablet so it exists + Map> ts3 = new HashMap>(); + ts3.put(mte2, new TreeMap()); + tservers.tservers.put("tserver3", ts3); + + assertNull(tab0TabletCache.locateTablet(new Text("row_0000000000"), false, false, credential)); + + } + + // this test reproduces a problem where empty metadata tablets, that were created by user tablets being merged away, caused locating tablets to fail + public void testBug3() throws Exception { + KeyExtent mte1 = new KeyExtent(new Text(Constants.METADATA_TABLE_ID), new Text("1;c"), RTE.getEndRow()); + KeyExtent mte2 = new KeyExtent(new Text(Constants.METADATA_TABLE_ID), new Text("1;f"), new Text("1;c")); + KeyExtent mte3 = new KeyExtent(new Text(Constants.METADATA_TABLE_ID), new Text("1;j"), new Text("1;f")); + KeyExtent mte4 = new KeyExtent(new Text(Constants.METADATA_TABLE_ID), new Text("1;r"), new Text("1;j")); + KeyExtent mte5 = new KeyExtent(new Text(Constants.METADATA_TABLE_ID), null, new Text("1;r")); + + KeyExtent ke1 = new KeyExtent(new Text("1"), null, null); + + TServers tservers = new TServers(); + TestTabletLocationObtainer ttlo = new TestTabletLocationObtainer(tservers); + TestInstance testInstance = new TestInstance("instance1", "tserver1"); + + RootTabletLocator rtl = new RootTabletLocator(testInstance); + + TabletLocatorImpl rootTabletCache = new TabletLocatorImpl(new Text(Constants.METADATA_TABLE_ID), rtl, ttlo); + TabletLocatorImpl tab0TabletCache = new TabletLocatorImpl(new Text("1"), rootTabletCache, ttlo); + + setLocation(tservers, "tserver1", RTE, mte1, "tserver2"); + setLocation(tservers, "tserver1", RTE, mte2, "tserver3"); + setLocation(tservers, "tserver1", RTE, mte3, "tserver4"); + setLocation(tservers, "tserver1", RTE, mte4, "tserver5"); + setLocation(tservers, "tserver1", RTE, mte5, "tserver6"); + + createEmptyTablet(tservers, "tserver2", mte1); + createEmptyTablet(tservers, "tserver3", mte2); + createEmptyTablet(tservers, "tserver4", mte3); + createEmptyTablet(tservers, "tserver5", mte4); + setLocation(tservers, "tserver6", mte5, ke1, "tserver7"); + + locateTabletTest(tab0TabletCache, "a", ke1, "tserver7", credential); + + } + + public void testAccumulo1248() throws Exception { + TServers tservers = new TServers(); + TabletLocatorImpl metaCache = createLocators(tservers, "tserver1", "tserver2", "foo"); + + KeyExtent ke1 = nke("foo", null, null); + + // set two locations for a tablet, this is not supposed to happen. The metadata cache should throw an exception if it sees this rather than caching one of + // the locations. + setLocation(tservers, "tserver2", MTE, ke1, "L1", "I1"); + setLocation(tservers, "tserver2", MTE, ke1, "L2", "I2"); + + try { + metaCache.locateTablet(new Text("a"), false, false, credential); + assertTrue(false); + } catch (Exception e) { + + } + + + } +}