Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7F3881047B for ; Mon, 2 Dec 2013 20:46:15 +0000 (UTC) Received: (qmail 69629 invoked by uid 500); 2 Dec 2013 20:46:15 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 69605 invoked by uid 500); 2 Dec 2013 20:46:15 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 69598 invoked by uid 99); 2 Dec 2013 20:46:15 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 02 Dec 2013 20:46:15 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 1031291A800; Mon, 2 Dec 2013 20:46:15 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vines@apache.org To: commits@accumulo.apache.org Message-Id: <57638db66d214d198f219b9de9338e5e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: ACCUMULO-1556 Clarify initialization error for used fs Date: Mon, 2 Dec 2013 20:46:15 +0000 (UTC) Updated Branches: refs/heads/1.4.5-SNAPSHOT 82477f08a -> 1b3d071a8 ACCUMULO-1556 Clarify initialization error for used fs The Initialize class now generates clearer and more informative error messages if an initialized instance is discovered. The messages vary depending on whether instance.dfs.uri is used. The values of instance.dfs.uri and instance.dfs.dir are also logged. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1b3d071a Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1b3d071a Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1b3d071a Branch: refs/heads/1.4.5-SNAPSHOT Commit: 1b3d071a8b46617245a39d039a2fe9e31600fc2e Parents: 82477f0 Author: Bill Havanki Authored: Tue Nov 5 17:07:28 2013 -0500 Committer: John Vines Committed: Fri Nov 29 14:26:41 2013 -0500 ---------------------------------------------------------------------- pom.xml | 6 + src/server/pom.xml | 4 + .../apache/accumulo/server/util/Initialize.java | 60 +++++-- .../accumulo/server/util/InitializeTest.java | 155 +++++++++++++++++++ 4 files changed, 211 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b3d071a/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ac57b9d..d0b3041 100644 --- a/pom.xml +++ b/pom.xml @@ -708,6 +708,12 @@ test + org.easymock + easymock + 3.1 + test + + com.google.guava guava 14.0.1 http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b3d071a/src/server/pom.xml ---------------------------------------------------------------------- diff --git a/src/server/pom.xml b/src/server/pom.xml index cc55e90..672a6d3 100644 --- a/src/server/pom.xml +++ b/src/server/pom.xml @@ -101,6 +101,10 @@ commons-lang + org.easymock + easymock + + org.mortbay.jetty jetty http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b3d071a/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java ---------------------------------------------------------------------- diff --git a/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java b/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java index 51576fc..c6bb57a 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java +++ b/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java @@ -29,6 +29,7 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.KeyExtent; import org.apache.accumulo.core.data.Value; @@ -73,12 +74,29 @@ public class Initialize { private static boolean clearInstanceName = false; private static ConsoleReader reader = null; + private static IZooReaderWriter zoo = ZooReaderWriter.getInstance(); private static ConsoleReader getConsoleReader() throws IOException { if (reader == null) reader = new ConsoleReader(); return reader; } + /** + * Sets this class's ZooKeeper reader/writer. + * + * @param izoo reader/writer + */ + static void setZooReaderWriter(IZooReaderWriter izoo) { + zoo = izoo; + } + /** + * Gets this class's ZooKeeper reader/writer. + * + * @return reader/writer + */ + static IZooReaderWriter getZooReaderWriter() { + return zoo; + } private static HashMap initialMetadataConf = new HashMap(); static { @@ -107,20 +125,18 @@ public class Initialize { initialMetadataConf.put(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true"); } - public static boolean doInit(Configuration conf, FileSystem fs) throws IOException { - if (!ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_DFS_URI).equals("")) - log.info("Hadoop Filesystem is " + ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_DFS_URI)); - else - log.info("Hadoop Filesystem is " + FileSystem.getDefaultUri(conf)); + static boolean checkInit(Configuration conf, FileSystem fs, SiteConfiguration sconf) throws IOException { + String fsUri = fs.getUri().toString(); + log.info("Hadoop Filesystem is " + fsUri); log.info("Accumulo data dir is " + ServerConstants.getBaseDir()); - log.info("Zookeeper server is " + ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST)); + log.info("Zookeeper server is " + sconf.get(Property.INSTANCE_ZK_HOST)); log.info("Checking if Zookeeper is available. If this hangs, then you need to make sure zookeeper is running"); if (!zookeeperAvailable()) { log.fatal("Zookeeper needs to be up and running in order to init. Exiting ..."); return false; } - if (ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_SECRET).equals(Property.INSTANCE_SECRET.getDefaultValue())) { + if (sconf.get(Property.INSTANCE_SECRET).equals(Property.INSTANCE_SECRET.getDefaultValue())) { ConsoleReader c = getConsoleReader(); c.beep(); c.printNewline(); @@ -137,16 +153,34 @@ public class Initialize { c.printNewline(); } - UUID uuid = UUID.randomUUID(); - try { if (isInitialized(fs)) { - log.fatal("It appears this location was previously initialized, exiting ... "); + String instanceDfsDir = sconf.get(Property.INSTANCE_DFS_DIR); + log.fatal("It appears the directory " + fsUri + instanceDfsDir + " was previously initialized."); + String instanceDfsUri = sconf.get(Property.INSTANCE_DFS_URI); + if ("".equals(instanceDfsUri)) { + log.fatal("You are using the default URI for the filesystem. Set the property " + Property.INSTANCE_DFS_URI + " to use a different filesystem,"); + } else { + log.fatal("Change the property " + Property.INSTANCE_DFS_URI + " to use a different filesystem,"); + } + log.fatal("or change the property " + Property.INSTANCE_DFS_DIR + " to use a different directory."); + log.fatal("The current value of " + Property.INSTANCE_DFS_URI + " is |" + instanceDfsUri + "|"); + log.fatal("The current value of " + Property.INSTANCE_DFS_DIR + " is |" + instanceDfsDir + "|"); return false; } } catch (IOException e) { - throw new RuntimeException(e); + throw new IOException("Failed to check if filesystem already initialized", e); + } + + return true; + } + + public static boolean doInit(Configuration conf, FileSystem fs) throws IOException { + if (!checkInit(conf, fs, ServerConfiguration.getSiteConfiguration())) { + return false; } + + UUID uuid = UUID.randomUUID(); // prompt user for instance name and root password early, in case they // abort, we don't leave an inconsistent HDFS/ZooKeeper structure @@ -186,7 +220,6 @@ public class Initialize { * @return */ private static boolean zookeeperAvailable() { - IZooReaderWriter zoo = ZooReaderWriter.getInstance(); try { return zoo.exists("/"); } catch (KeeperException e) { @@ -332,7 +365,6 @@ public class Initialize { private static void initZooKeeper(String uuid, String instanceNamePath) throws KeeperException, InterruptedException { // setup basic data in zookeeper - IZooReaderWriter zoo = ZooReaderWriter.getInstance(); ZooUtil.putPersistentData(zoo.getZooKeeper(), Constants.ZROOT, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE); ZooUtil.putPersistentData(zoo.getZooKeeper(), Constants.ZROOT + Constants.ZINSTANCES, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE); @@ -378,7 +410,7 @@ public class Initialize { if (clearInstanceName) { exists = false; break; - } else if ((boolean) (exists = ZooReaderWriter.getInstance().exists(instanceNamePath))) { + } else if ((boolean) (exists = zoo.exists(instanceNamePath))) { String decision = getConsoleReader().readLine("Instance name \"" + instanceName + "\" exists. Delete existing entry from zookeeper? [Y/N] : "); if (decision == null) System.exit(0); http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b3d071a/src/server/src/test/java/org/apache/accumulo/server/util/InitializeTest.java ---------------------------------------------------------------------- diff --git a/src/server/src/test/java/org/apache/accumulo/server/util/InitializeTest.java b/src/server/src/test/java/org/apache/accumulo/server/util/InitializeTest.java new file mode 100644 index 0000000..138a0ac --- /dev/null +++ b/src/server/src/test/java/org/apache/accumulo/server/util/InitializeTest.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.server.util; + +import java.io.IOException; +import java.net.URI; + +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.conf.SiteConfiguration; +import org.apache.accumulo.server.zookeeper.IZooReaderWriter; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.easymock.Capture; +import static org.easymock.EasyMock.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * This test is not thread-safe. + */ +public class InitializeTest { + private Configuration conf; + private FileSystem fs; + private SiteConfiguration sconf; + private IZooReaderWriter zooOrig; + private IZooReaderWriter zoo; + + @Before + public void setUp() throws Exception { + conf = createMock(Configuration.class); + fs = createMock(FileSystem.class); + sconf = createMock(SiteConfiguration.class); + zoo = createMock(IZooReaderWriter.class); + zooOrig = Initialize.getZooReaderWriter(); + Initialize.setZooReaderWriter(zoo); + } + + @After + public void tearDown() { + Initialize.setZooReaderWriter(zooOrig); + } + + @Test + public void testIsInitialized_HasInstanceId() throws Exception { + expect(fs.exists(anyObject(Path.class))).andReturn(true); + replay(fs); + assertTrue(Initialize.isInitialized(fs)); + } + + @Test + public void testIsInitialized_HasDataVersion() throws Exception { + expect(fs.exists(anyObject(Path.class))).andReturn(false); + expect(fs.exists(anyObject(Path.class))).andReturn(true); + replay(fs); + assertTrue(Initialize.isInitialized(fs)); + } + + @Test + public void testCheckInit_NoZK() throws Exception { + expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn("hdfs://foo"); + expectLastCall().anyTimes(); + expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1"); + replay(sconf); + expect(zoo.exists("/")).andReturn(false); + replay(zoo); + expect(fs.getUri()).andReturn(new URI("hdfs://foo")); + replay(fs); + + assertFalse(Initialize.checkInit(conf, fs, sconf)); + } + + @Test + public void testCheckInit_AlreadyInit() throws Exception { + expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn("hdfs://foo"); + expectLastCall().anyTimes(); + expect(sconf.get(Property.INSTANCE_DFS_DIR)).andReturn("/bar"); + expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1"); + expect(sconf.get(Property.INSTANCE_SECRET)).andReturn(Property.INSTANCE_SECRET.getDefaultValue()); + replay(sconf); + expect(zoo.exists("/")).andReturn(true); + replay(zoo); + expect(fs.getUri()).andReturn(new URI("hdfs://foo")); + expect(fs.exists(anyObject(Path.class))).andReturn(true); + replay(fs); + + assertFalse(Initialize.checkInit(conf, fs, sconf)); + } + + @Test + public void testCheckInit_AlreadyInit_DefaultUri() throws Exception { + expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn(""); + expectLastCall().anyTimes(); + expect(sconf.get(Property.INSTANCE_DFS_DIR)).andReturn("/bar"); + expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1"); + expect(sconf.get(Property.INSTANCE_SECRET)).andReturn(Property.INSTANCE_SECRET.getDefaultValue()); + replay(sconf); + expect(zoo.exists("/")).andReturn(true); + replay(zoo); + expect(fs.getUri()).andReturn(new URI("hdfs://default")); + expect(fs.exists(anyObject(Path.class))).andReturn(true); + replay(fs); + + assertFalse(Initialize.checkInit(conf, fs, sconf)); + } + + @Test(expected = IOException.class) + public void testCheckInit_FSException() throws Exception { + expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn("hdfs://foo"); + expectLastCall().anyTimes(); + expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1"); + expect(sconf.get(Property.INSTANCE_SECRET)).andReturn(Property.INSTANCE_SECRET.getDefaultValue()); + replay(sconf); + expect(zoo.exists("/")).andReturn(true); + replay(zoo); + expect(fs.getUri()).andReturn(new URI("hdfs://foo")); + expect(fs.exists(anyObject(Path.class))).andThrow(new IOException()); + replay(fs); + + Initialize.checkInit(conf, fs, sconf); + } + + @Test + public void testCheckInit_OK() throws Exception { + expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn("hdfs://foo"); + expectLastCall().anyTimes(); + expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1"); + expect(sconf.get(Property.INSTANCE_SECRET)).andReturn(Property.INSTANCE_SECRET.getDefaultValue()); + replay(sconf); + expect(zoo.exists("/")).andReturn(true); + replay(zoo); + expect(fs.getUri()).andReturn(new URI("hdfs://foo")); + expect(fs.exists(anyObject(Path.class))).andReturn(false); + expect(fs.exists(anyObject(Path.class))).andReturn(false); + replay(fs); + + assertTrue(Initialize.checkInit(conf, fs, sconf)); + } +}