Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2E82E10AE5 for ; Tue, 28 Jan 2014 17:12:58 +0000 (UTC) Received: (qmail 72117 invoked by uid 500); 28 Jan 2014 17:12:57 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 72053 invoked by uid 500); 28 Jan 2014 17:12:57 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 72046 invoked by uid 99); 28 Jan 2014 17:12:57 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 28 Jan 2014 17:12:57 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 04CD290A215; Tue, 28 Jan 2014 17:12:56 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: kturner@apache.org To: commits@accumulo.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: git commit: ACCUMULO-1772 If at table load time volume is no longer configured, then choose another Date: Tue, 28 Jan 2014 17:12:56 +0000 (UTC) Updated Branches: refs/heads/1.6.0-SNAPSHOT 3c3911dd5 -> 783e895fb ACCUMULO-1772 If at table load time volume is no longer configured, then choose another Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/783e895f Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/783e895f Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/783e895f Branch: refs/heads/1.6.0-SNAPSHOT Commit: 783e895fbab086059c0a3d64f8fba566973ffcc6 Parents: 3c3911d Author: Keith Turner Authored: Tue Jan 28 12:17:28 2014 -0500 Committer: Keith Turner Committed: Tue Jan 28 12:17:28 2014 -0500 ---------------------------------------------------------------------- .../accumulo/server/util/MetadataTableUtil.java | 20 +++ .../tserver/DirectoryDecommissioner.java | 169 +++++++++++++++++++ .../org/apache/accumulo/tserver/Tablet.java | 6 +- .../tserver/DirectoryDecommissionerTest.java | 108 ++++++++++++ .../java/org/apache/accumulo/test/VolumeIT.java | 81 +++++++-- 5 files changed, 365 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/783e895f/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java index 158157b..bc80b9e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java @@ -177,6 +177,12 @@ public class MetadataTableUtil { update(credentials, zooLock, m, extent); } + public static void updateTabletDir(KeyExtent extent, String newDir, Credentials creds, ZooLock lock) { + Mutation m = new Mutation(extent.getMetadataEntry()); + TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes())); + update(creds, lock, m, extent); + } + public static void addTablet(KeyExtent extent, String path, Credentials credentials, char timeType, ZooLock lock) { Mutation m = extent.getPrevRowUpdateMutation(); @@ -418,6 +424,19 @@ public class MetadataTableUtil { } } + public static void setRootTabletDir(String dir) throws IOException { + IZooReaderWriter zoo = ZooReaderWriter.getInstance(); + String zpath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_PATH; + try { + zoo.putPersistentData(zpath, dir.getBytes(Constants.UTF8), -1, NodeExistsPolicy.OVERWRITE); + } catch (KeeperException e) { + throw new IOException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } + } + public static String getRootTabletDir() throws IOException { IZooReaderWriter zoo = ZooReaderWriter.getInstance(); String zpath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_PATH; @@ -426,6 +445,7 @@ public class MetadataTableUtil { } catch (KeeperException e) { throw new IOException(e); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new IOException(e); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/783e895f/server/tserver/src/main/java/org/apache/accumulo/tserver/DirectoryDecommissioner.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/DirectoryDecommissioner.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/DirectoryDecommissioner.java new file mode 100644 index 0000000..ea932ff --- /dev/null +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/DirectoryDecommissioner.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.tserver; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.security.SecureRandom; +import java.util.HashSet; + +import org.apache.accumulo.core.data.KeyExtent; +import org.apache.accumulo.core.util.CachedConfiguration; +import org.apache.accumulo.server.ServerConstants; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.security.SystemCredentials; +import org.apache.accumulo.server.util.MetadataTableUtil; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.log4j.Logger; + +/** + * This class contains utility code for switching a tablets default directory, if that default directory is no longer configured. + */ +public class DirectoryDecommissioner { + + private static final Logger log = Logger.getLogger(DirectoryDecommissioner.class); + + public static boolean isActiveVolume(Path dir) { + for (String tableDir : ServerConstants.getTablesDirs()) { + // use Path to normalize tableDir + if (dir.toString().startsWith(new Path(tableDir).toString())) + return true; + } + + return false; + } + + public static Path checkTabletDirectory(TabletServer tserver, VolumeManager vm, KeyExtent extent, Path dir) throws IOException { + if (isActiveVolume(dir)) + return dir; + + if (!dir.getParent().getParent().getName().equals(ServerConstants.TABLE_DIR)) { + throw new IllegalArgumentException("Unexpected table dir " + dir); + } + + Path newDir = new Path(vm.choose(ServerConstants.getTablesDirs()) + "/" + dir.getParent().getName() + "/" + dir.getName()); + + log.info("Updating directory for " + extent + " from " + dir + " to " + newDir); + if (extent.isRootTablet()) { + // the root tablet is special case, its files need to be copied if its dir is changed + + // this code needs to be idempotent + + FileSystem fs1 = vm.getFileSystemByPath(dir); + FileSystem fs2 = vm.getFileSystemByPath(newDir); + + if (!same(fs1, dir, fs2, newDir)) { + if (fs2.exists(newDir)) { + Path newDirBackup = getBackupName(fs2, newDir); + // never delete anything because were dealing with the root tablet + // one reason this dir may exist is because this method failed previously + log.info("renaming " + newDir + " to " + newDirBackup); + if (!fs2.rename(newDir, newDirBackup)) { + throw new IOException("Failed to rename " + newDir + " to " + newDirBackup); + } + } + + // do a lot of logging since this is the root tablet + log.info("copying " + dir + " to " + newDir); + if (!FileUtil.copy(fs1, dir, fs2, newDir, false, CachedConfiguration.getInstance())) { + throw new IOException("Failed to copy " + dir + " to " + newDir); + } + + // only set the new location in zookeeper after a successful copy + log.info("setting root tablet location to " + newDir); + MetadataTableUtil.setRootTabletDir(newDir.toString()); + + // rename the old dir to avoid confusion when someone looks at filesystem... its ok if we fail here and this does not happen because the location in + // zookeeper is the authority + Path dirBackup = getBackupName(fs1, dir); + log.info("renaming " + dir + " to " + dirBackup); + fs1.rename(dir, dirBackup); + } else { + log.info("setting root tablet location to " + newDir); + MetadataTableUtil.setRootTabletDir(newDir.toString()); + } + + return dir; + } else { + MetadataTableUtil.updateTabletDir(extent, newDir.toString(), SystemCredentials.get(), tserver.getLock()); + return newDir; + } + } + + static boolean same(FileSystem fs1, Path dir, FileSystem fs2, Path newDir) throws FileNotFoundException, IOException { + // its possible that a user changes config in such a way that two uris point to the same thing. Like hdfs://foo/a/b and hdfs://1.2.3.4/a/b both reference + // the same thing because DNS resolves foo to 1.2.3.4. This method does not analyze uris to determine if equivalent, instead it inspects the contents of + // what the uris point to. + + //this code is called infrequently and does not need to be optimized. + + if (fs1.exists(dir) && fs2.exists(newDir)) { + + if (!fs1.isDirectory(dir)) + throw new IllegalArgumentException("expected " + dir + " to be a directory"); + + + if (!fs2.isDirectory(newDir)) + throw new IllegalArgumentException("expected " + newDir + " to be a directory"); + + + HashSet names1 = getFileNames(fs1.listStatus(dir)); + HashSet names2 = getFileNames(fs2.listStatus(newDir)); + + if (names1.equals(names2)) { + for (String name : names1) + if (!hash(fs1, dir, name).equals(hash(fs2, newDir, name))) + return false; + return true; + } + + } + return false; + } + + @SuppressWarnings("deprecation") + private static HashSet getFileNames(FileStatus[] filesStatuses) { + HashSet names = new HashSet(); + for (FileStatus fileStatus : filesStatuses) + if (fileStatus.isDir()) + throw new IllegalArgumentException("expected " + fileStatus.getPath() + " to be a file"); + else + names.add(fileStatus.getPath().getName()); + return names; + } + + private static String hash(FileSystem fs, Path dir, String name) throws IOException { + FSDataInputStream in = fs.open(new Path(dir, name)); + try { + return DigestUtils.sha1Hex(in); + } finally { + in.close(); + } + + } + + private static Path getBackupName(FileSystem fs, Path path) { + SecureRandom rand = new SecureRandom(); + return new Path(path.getParent(), path.getName() + "_" + System.currentTimeMillis() + "_" + Math.abs(rand.nextInt()) + ".bak"); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/783e895f/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java index e457ed5..d3f8993 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java @@ -1256,8 +1256,10 @@ public class Tablet { } else { locationPath = fs.getFullPath(FileType.TABLE, extent.getTableId().toString() + location.toString()); } - FileSystem fsForPath = fs.getFileSystemByPath(locationPath); - this.location = locationPath.makeQualified(fsForPath.getUri(), fsForPath.getWorkingDirectory()); + + locationPath = DirectoryDecommissioner.checkTabletDirectory(tabletServer, fs, extent, locationPath); + + this.location = locationPath; this.lastLocation = lastLocation; this.tabletDirectory = location.toString(); this.conf = conf; http://git-wip-us.apache.org/repos/asf/accumulo/blob/783e895f/server/tserver/src/test/java/org/apache/accumulo/tserver/DirectoryDecommissionerTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/DirectoryDecommissionerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/DirectoryDecommissionerTest.java new file mode 100644 index 0000000..47cdab9 --- /dev/null +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/DirectoryDecommissionerTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.tserver; + +import java.io.File; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * + */ +public class DirectoryDecommissionerTest { + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target")); + + @Test + public void testSame() throws Exception { + FileSystem fs = FileSystem.getLocal(new Configuration()); + + Path subdir1 = new Path(tempFolder.newFolder().toURI()); + Path subdir2 = new Path(tempFolder.newFolder().toURI()); + Path subdir3 = new Path(tempFolder.newFolder().toURI()); + + Assert.assertFalse(DirectoryDecommissioner.same(fs, subdir1, fs, new Path(tempFolder.getRoot().toURI().toString(), "8854339269459287524098238497"))); + Assert.assertFalse(DirectoryDecommissioner.same(fs, new Path(tempFolder.getRoot().toURI().toString(), "8854339269459287524098238497"), fs, subdir1)); + Assert.assertTrue(DirectoryDecommissioner.same(fs, subdir1, fs, subdir1)); + + writeFile(fs, subdir1, "abc", "foo"); + writeFile(fs, subdir2, "abc", "bar"); + writeFile(fs, subdir3, "abc", "foo"); + + Assert.assertTrue(DirectoryDecommissioner.same(fs, subdir1, fs, subdir1)); + Assert.assertFalse(DirectoryDecommissioner.same(fs, subdir1, fs, subdir2)); + Assert.assertFalse(DirectoryDecommissioner.same(fs, subdir2, fs, subdir1)); + Assert.assertTrue(DirectoryDecommissioner.same(fs, subdir1, fs, subdir3)); + Assert.assertTrue(DirectoryDecommissioner.same(fs, subdir3, fs, subdir1)); + + writeFile(fs, subdir1, "def", "123456"); + writeFile(fs, subdir2, "def", "123456"); + writeFile(fs, subdir3, "def", "123456"); + + Assert.assertTrue(DirectoryDecommissioner.same(fs, subdir1, fs, subdir1)); + Assert.assertFalse(DirectoryDecommissioner.same(fs, subdir1, fs, subdir2)); + Assert.assertFalse(DirectoryDecommissioner.same(fs, subdir2, fs, subdir1)); + Assert.assertTrue(DirectoryDecommissioner.same(fs, subdir1, fs, subdir3)); + Assert.assertTrue(DirectoryDecommissioner.same(fs, subdir3, fs, subdir1)); + + writeFile(fs, subdir3, "ghi", "09876"); + + Assert.assertFalse(DirectoryDecommissioner.same(fs, subdir1, fs, subdir3)); + Assert.assertFalse(DirectoryDecommissioner.same(fs, subdir3, fs, subdir1)); + + fs.mkdirs(new Path(subdir2, "dir1")); + + try { + DirectoryDecommissioner.same(fs, subdir1, fs, subdir2); + Assert.fail(); + } catch (IllegalArgumentException e) {} + + try { + DirectoryDecommissioner.same(fs, subdir2, fs, subdir1); + Assert.fail(); + } catch (IllegalArgumentException e) {} + + try { + DirectoryDecommissioner.same(fs, subdir1, fs, new Path(subdir2, "def")); + Assert.fail(); + } catch (IllegalArgumentException e) {} + + try { + DirectoryDecommissioner.same(fs, new Path(subdir2, "def"), fs, subdir3); + Assert.fail(); + } catch (IllegalArgumentException e) {} + + } + + private void writeFile(FileSystem fs, Path dir, String filename, String data) throws IOException { + FSDataOutputStream out = fs.create(new Path(dir, filename)); + try { + out.writeUTF(data); + } finally { + out.close(); + } + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/783e895f/test/src/test/java/org/apache/accumulo/test/VolumeIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java index 2f64d58..ee38efd 100644 --- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java +++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java @@ -31,6 +31,7 @@ import java.util.Map.Entry; import java.util.SortedSet; import java.util.TreeSet; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; @@ -49,11 +50,13 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.schema.MetadataSchema; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.TablePermission; import org.apache.accumulo.core.util.CachedConfiguration; +import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; import org.apache.accumulo.server.ServerConstants; import org.apache.accumulo.server.init.Initialize; @@ -65,6 +68,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; +import org.apache.zookeeper.ZooKeeper; import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; @@ -298,33 +302,40 @@ public class VolumeIT extends ConfigurableMacIT { } private void verifyVolumesUsed(String tableName, Path... paths) throws AccumuloException, AccumuloSecurityException, TableExistsException, - TableNotFoundException, - MutationsRejectedException { - TreeSet splits = new TreeSet(); - for (int i = 0; i < 100; i++) { - splits.add(new Text(String.format("%06d", i * 100))); - } + TableNotFoundException, MutationsRejectedException { Connector conn = cluster.getConnector("root", ROOT_PASSWORD); - conn.tableOperations().create(tableName); - conn.tableOperations().addSplits(tableName, splits); List expected = new ArrayList(); - - BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig()); for (int i = 0; i < 100; i++) { String row = String.format("%06d", i * 100 + 3); - Mutation m = new Mutation(row); - m.put("cf1", "cq1", "1"); - bw.addMutation(m); expected.add(row + ":cf1:cq1:1"); } - bw.close(); + if (!conn.tableOperations().exists(tableName)) { - verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY)); + TreeSet splits = new TreeSet(); + for (int i = 0; i < 100; i++) { + splits.add(new Text(String.format("%06d", i * 100))); + } - conn.tableOperations().flush(tableName, null, null, true); + conn.tableOperations().create(tableName); + conn.tableOperations().addSplits(tableName, splits); + + BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig()); + for (int i = 0; i < 100; i++) { + String row = String.format("%06d", i * 100 + 3); + Mutation m = new Mutation(row); + m.put("cf1", "cq1", "1"); + bw.addMutation(m); + } + + bw.close(); + + verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY)); + + conn.tableOperations().flush(tableName, null, null, true); + } verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY)); @@ -335,13 +346,16 @@ public class VolumeIT extends ConfigurableMacIT { int counts[] = new int[paths.length]; - for (Entry entry : metaScanner) { + outer: for (Entry entry : metaScanner) { String cq = entry.getKey().getColumnQualifier().toString(); for (int i = 0; i < paths.length; i++) { if (cq.startsWith(paths[i].toString())) { counts[i]++; + continue outer; } } + + Assert.fail("Unexpected volume " + cq); } // if a volume is chosen randomly for each tablet, then the probability that a volume will not be chosen for any tablet is ((num_volumes - @@ -356,4 +370,37 @@ public class VolumeIT extends ConfigurableMacIT { Assert.assertEquals(100, sum); } + @Test + public void testRemoveVolumes() throws Exception { + String[] tableNames = getTableNames(1); + + verifyVolumesUsed(tableNames[0], v1, v2); + + Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor()); + cluster.stop(); + + Configuration conf = new Configuration(false); + conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml")); + + conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString()); + BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml"))); + conf.writeXml(fos); + fos.close(); + + // start cluster and verify that volume was decommisioned + cluster.start(); + + Connector conn = cluster.getConnector("root", ROOT_PASSWORD); + conn.tableOperations().compact(tableNames[0], null, null, true, true); + + verifyVolumesUsed(tableNames[0], v2); + + // check that root tablet is not on volume 1 + String zpath = ZooUtil.getRoot(new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers())) + RootTable.ZROOT_TABLET_PATH; + ZooKeeper zookeeper = new ZooKeeper(cluster.getZooKeepers(), 30000, null); + String rootTabletDir = new String(zookeeper.getData(zpath, false, null), Constants.UTF8); + Assert.assertTrue(rootTabletDir.startsWith(v2.toString())); + zookeeper.close(); + + } }