Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E2D3C10497 for ; Mon, 2 Dec 2013 20:46:54 +0000 (UTC) Received: (qmail 73052 invoked by uid 500); 2 Dec 2013 20:46:54 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 72998 invoked by uid 500); 2 Dec 2013 20:46:54 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 72918 invoked by uid 99); 2 Dec 2013 20:46:54 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 02 Dec 2013 20:46:54 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 8534491A864; Mon, 2 Dec 2013 20:46:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vines@apache.org To: commits@accumulo.apache.org Date: Mon, 02 Dec 2013 20:46:58 -0000 Message-Id: <8c4947979a1749f6aebcf06fe3615384@git.apache.org> In-Reply-To: <0bceeacea16e4f5eb30cce893d99c82a@git.apache.org> References: <0bceeacea16e4f5eb30cce893d99c82a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [5/6] git commit: ACCUMULO-1556 Clarify initialization error for used fs ACCUMULO-1556 Clarify initialization error for used fs The Initialize class now generates clearer and more informative error messages if an initialized instance is discovered. The messages vary depending on whether instance.dfs.uri is used. The values of instance.dfs.uri and instance.dfs.dir are also logged. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/31b58c47 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/31b58c47 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/31b58c47 Branch: refs/heads/master Commit: 31b58c473d4abd4c7a9f51c558bdc89b61a92bfe Parents: cb2ec9b Author: Bill Havanki Authored: Mon Dec 2 10:35:35 2013 -0500 Committer: John Vines Committed: Mon Dec 2 15:45:29 2013 -0500 ---------------------------------------------------------------------- server/base/pom.xml | 5 + .../apache/accumulo/server/init/Initialize.java | 63 ++++++-- .../accumulo/server/init/InitializeTest.java | 152 +++++++++++++++++++ 3 files changed, 205 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/31b58c47/server/base/pom.xml ---------------------------------------------------------------------- diff --git a/server/base/pom.xml b/server/base/pom.xml index 69c766e..3e59fb8 100644 --- a/server/base/pom.xml +++ b/server/base/pom.xml @@ -96,6 +96,11 @@ junit test + + org.easymock + easymock + test + http://git-wip-us.apache.org/repos/asf/accumulo/blob/31b58c47/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java index 0477a44..2ae0763 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java +++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java @@ -87,13 +87,30 @@ public class Initialize { public static final String TABLE_TABLETS_TABLET_DIR = "/table_info"; private static ConsoleReader reader = null; + private static IZooReaderWriter zoo = ZooReaderWriter.getInstance(); private static ConsoleReader getConsoleReader() throws IOException { if (reader == null) reader = new ConsoleReader(); return reader; } - + /** + * Sets this class's ZooKeeper reader/writer. + * + * @param izoo reader/writer + */ + static void setZooReaderWriter(IZooReaderWriter izoo) { + zoo = izoo; + } + /** + * Gets this class's ZooKeeper reader/writer. + * + * @return reader/writer + */ + static IZooReaderWriter getZooReaderWriter() { + return zoo; + } + private static HashMap initialMetadataConf = new HashMap(); static { initialMetadataConf.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "32K"); @@ -120,20 +137,21 @@ public class Initialize { initialMetadataConf.put(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true"); } - public static boolean doInit(Opts opts, Configuration conf, VolumeManager fs) throws IOException { - if (!ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_DFS_URI).equals("")) - log.info("Hadoop Filesystem is " + ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_DFS_URI)); + static boolean checkInit(Configuration conf, VolumeManager fs, SiteConfiguration sconf) throws IOException { + String fsUri; + if (!sconf.get(Property.INSTANCE_DFS_URI).equals("")) + fsUri = sconf.get(Property.INSTANCE_DFS_URI); else - log.info("Hadoop Filesystem is " + FileSystem.getDefaultUri(conf)); - + fsUri = FileSystem.getDefaultUri(conf).toString(); + log.info("Hadoop Filesystem is " + fsUri); log.info("Accumulo data dirs are " + Arrays.asList(ServerConstants.getBaseDirs())); - log.info("Zookeeper server is " + ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST)); + log.info("Zookeeper server is " + sconf.get(Property.INSTANCE_ZK_HOST)); log.info("Checking if Zookeeper is available. If this hangs, then you need to make sure zookeeper is running"); if (!zookeeperAvailable()) { log.fatal("Zookeeper needs to be up and running in order to init. Exiting ..."); return false; } - if (ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_SECRET).equals(Property.INSTANCE_SECRET.getDefaultValue())) { + if (sconf.get(Property.INSTANCE_SECRET).equals(Property.INSTANCE_SECRET.getDefaultValue())) { ConsoleReader c = getConsoleReader(); c.beep(); c.println(); @@ -145,16 +163,33 @@ public class Initialize { c.println(" bin/accumulo " + org.apache.accumulo.server.util.ChangeSecret.class.getName() + " oldPassword newPassword."); c.println("You will also need to edit your secret in your configuration file by adding the property instance.secret to your conf/accumulo-site.xml. Without this accumulo will not operate correctly"); } - try { if (isInitialized(fs)) { - log.fatal("It appears this location was previously initialized, exiting ... "); + String instanceDfsDir = sconf.get(Property.INSTANCE_DFS_DIR); + log.fatal("It appears the directory " + fsUri + instanceDfsDir + " was previously initialized."); + String instanceDfsUri = sconf.get(Property.INSTANCE_DFS_URI); + if ("".equals(instanceDfsUri)) { + log.fatal("You are using the default URI for the filesystem. Set the property " + Property.INSTANCE_DFS_URI + " to use a different filesystem,"); + } else { + log.fatal("Change the property " + Property.INSTANCE_DFS_URI + " to use a different filesystem,"); + } + log.fatal("or change the property " + Property.INSTANCE_DFS_DIR + " to use a different directory."); + log.fatal("The current value of " + Property.INSTANCE_DFS_URI + " is |" + instanceDfsUri + "|"); + log.fatal("The current value of " + Property.INSTANCE_DFS_DIR + " is |" + instanceDfsDir + "|"); return false; } } catch (IOException e) { - throw new RuntimeException(e); + throw new IOException("Failed to check if filesystem already initialized", e); } - + + return true; + } + + public static boolean doInit(Opts opts, Configuration conf, VolumeManager fs) throws IOException { + if (!checkInit(conf, fs, ServerConfiguration.getSiteConfiguration())) { + return false; + } + // prompt user for instance name and root password early, in case they // abort, we don't leave an inconsistent HDFS/ZooKeeper structure String instanceNamePath; @@ -197,7 +232,6 @@ public class Initialize { } private static boolean zookeeperAvailable() { - IZooReaderWriter zoo = ZooReaderWriter.getInstance(); try { return zoo.exists("/"); } catch (KeeperException e) { @@ -345,7 +379,6 @@ public class Initialize { private static void initZooKeeper(Opts opts, String uuid, String instanceNamePath, Path rootTablet) throws KeeperException, InterruptedException { // setup basic data in zookeeper - IZooReaderWriter zoo = ZooReaderWriter.getInstance(); ZooUtil.putPersistentData(zoo.getZooKeeper(), Constants.ZROOT, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE); ZooUtil.putPersistentData(zoo.getZooKeeper(), Constants.ZROOT + Constants.ZINSTANCES, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE); @@ -397,7 +430,7 @@ public class Initialize { if (opts.clearInstanceName) { exists = false; break; - } else if (exists = ZooReaderWriter.getInstance().exists(instanceNamePath)) { + } else if (exists = zoo.exists(instanceNamePath)) { String decision = getConsoleReader().readLine("Instance name \"" + instanceName + "\" exists. Delete existing entry from zookeeper? [Y/N] : "); if (decision == null) System.exit(0); http://git-wip-us.apache.org/repos/asf/accumulo/blob/31b58c47/server/base/src/test/java/org/apache/accumulo/server/init/InitializeTest.java ---------------------------------------------------------------------- diff --git a/server/base/src/test/java/org/apache/accumulo/server/init/InitializeTest.java b/server/base/src/test/java/org/apache/accumulo/server/init/InitializeTest.java new file mode 100644 index 0000000..6d0f417 --- /dev/null +++ b/server/base/src/test/java/org/apache/accumulo/server/init/InitializeTest.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.server.init; + +import java.io.IOException; +import java.net.URI; + +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.conf.SiteConfiguration; +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.easymock.Capture; +import static org.easymock.EasyMock.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * This test is not thread-safe. + */ +public class InitializeTest { + private Configuration conf; + private VolumeManager fs; + private SiteConfiguration sconf; + private IZooReaderWriter zooOrig; + private IZooReaderWriter zoo; + + @Before + public void setUp() throws Exception { + conf = createMock(Configuration.class); + fs = createMock(VolumeManager.class); + sconf = createMock(SiteConfiguration.class); + zoo = createMock(IZooReaderWriter.class); + zooOrig = Initialize.getZooReaderWriter(); + Initialize.setZooReaderWriter(zoo); + } + + @After + public void tearDown() { + Initialize.setZooReaderWriter(zooOrig); + } + + @Test + public void testIsInitialized_HasInstanceId() throws Exception { + expect(fs.exists(anyObject(Path.class))).andReturn(true); + replay(fs); + assertTrue(Initialize.isInitialized(fs)); + } + + @Test + public void testIsInitialized_HasDataVersion() throws Exception { + expect(fs.exists(anyObject(Path.class))).andReturn(false); + expect(fs.exists(anyObject(Path.class))).andReturn(true); + replay(fs); + assertTrue(Initialize.isInitialized(fs)); + } + + @Test + public void testCheckInit_NoZK() throws Exception { + expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn("hdfs://foo"); + expectLastCall().anyTimes(); + expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1"); + replay(sconf); + expect(zoo.exists("/")).andReturn(false); + replay(zoo); + + assertFalse(Initialize.checkInit(conf, fs, sconf)); + } + + @Test + public void testCheckInit_AlreadyInit() throws Exception { + expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn("hdfs://foo"); + expectLastCall().anyTimes(); + expect(sconf.get(Property.INSTANCE_DFS_DIR)).andReturn("/bar"); + expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1"); + expect(sconf.get(Property.INSTANCE_SECRET)).andReturn(Property.INSTANCE_SECRET.getDefaultValue()); + replay(sconf); + expect(zoo.exists("/")).andReturn(true); + replay(zoo); + expect(fs.exists(anyObject(Path.class))).andReturn(true); + replay(fs); + + assertFalse(Initialize.checkInit(conf, fs, sconf)); + } + +/* Cannot test, need to mock static FileSystem.getDefaultUri() + @Test + public void testCheckInit_AlreadyInit_DefaultUri() throws Exception { + expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn(""); + expectLastCall().anyTimes(); + expect(sconf.get(Property.INSTANCE_DFS_DIR)).andReturn("/bar"); + expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1"); + expect(sconf.get(Property.INSTANCE_SECRET)).andReturn(Property.INSTANCE_SECRET.getDefaultValue()); + replay(sconf); + expect(zoo.exists("/")).andReturn(true); + replay(zoo); + // expect(fs.getUri()).andReturn(new URI("hdfs://default")); + expect(fs.exists(anyObject(Path.class))).andReturn(true); + replay(fs); + + assertFalse(Initialize.checkInit(conf, fs, sconf)); + } +*/ + + @Test(expected = IOException.class) + public void testCheckInit_FSException() throws Exception { + expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn("hdfs://foo"); + expectLastCall().anyTimes(); + expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1"); + expect(sconf.get(Property.INSTANCE_SECRET)).andReturn(Property.INSTANCE_SECRET.getDefaultValue()); + replay(sconf); + expect(zoo.exists("/")).andReturn(true); + replay(zoo); + expect(fs.exists(anyObject(Path.class))).andThrow(new IOException()); + replay(fs); + + Initialize.checkInit(conf, fs, sconf); + } + + @Test + public void testCheckInit_OK() throws Exception { + expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn("hdfs://foo"); + expectLastCall().anyTimes(); + expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1"); + expect(sconf.get(Property.INSTANCE_SECRET)).andReturn(Property.INSTANCE_SECRET.getDefaultValue()); + replay(sconf); + expect(zoo.exists("/")).andReturn(true); + replay(zoo); + expect(fs.exists(anyObject(Path.class))).andReturn(false); + expect(fs.exists(anyObject(Path.class))).andReturn(false); + replay(fs); + + assertTrue(Initialize.checkInit(conf, fs, sconf)); + } +}