accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cjno...@apache.org
Subject accumulo git commit: ACCUMULO-2793 Adding documentation to administration section of user manual for migrating from a non-HA volume to an HA volume. Also adding log error when the second pair item in an instance.volumes.replacements entry appears in the
Date Wed, 21 Jan 2015 01:00:36 GMT
Repository: accumulo
Updated Branches:
  refs/heads/1.6 c5153331c -> 21ab05578


ACCUMULO-2793 Adding documentation to administration section of user manual for migrating
from a non-HA volume to an HA volume. Also adding log error when the second pair item in an
instance.volumes.replacements entry appears in the instance.volumes property during the "add
volumes" initialize command.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/21ab0557
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/21ab0557
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/21ab0557

Branch: refs/heads/1.6
Commit: 21ab0557828394891d574567bc529867d11012e3
Parents: c515333
Author: Corey J. Nolet <cjnolet@gmail.com>
Authored: Thu Jan 15 22:12:27 2015 -0500
Committer: Corey J. Nolet <cjnolet@gmail.com>
Committed: Tue Jan 20 20:00:15 2015 -0500

----------------------------------------------------------------------
 .../chapters/administration.tex                 | 65 ++++++++++++++++++++
 .../apache/accumulo/server/ServerConstants.java |  1 +
 .../apache/accumulo/server/init/Initialize.java | 17 +++--
 3 files changed, 79 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/21ab0557/docs/src/main/latex/accumulo_user_manual/chapters/administration.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/administration.tex b/docs/src/main/latex/accumulo_user_manual/chapters/administration.tex
index 4b917d1..120e88b 100644
--- a/docs/src/main/latex/accumulo_user_manual/chapters/administration.tex
+++ b/docs/src/main/latex/accumulo_user_manual/chapters/administration.tex
@@ -539,3 +539,68 @@ The Accumulo client library is configured to retry failed mutations and
in many
 cases clients will be able to continue processing after the recovery process without
 throwing an exception.
 
+\section{Migrating Accumulo from non-HA Namenode to HA Namenode}
+
+The following steps will allow a non-HA instance to be migrated to an HA instance. Consider
an HDFS URL
+\texttt{hdfs://namenode.example.com:8020} which is going to be moved to \texttt{hdfs://nameservice1}.
+
+Before moving HDFS over to the HA namenode, use \texttt{\$ACCUMULO\_HOME/bin/accumulo admin
volumes} to confirm
+that the only volume displayed is the volume from the current namenode's HDFS URL.
+
+\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
+Listing volumes referenced in zookeeper
+        Volume : hdfs://namenode.example.com:8020/accumulo
+
+Listing volumes referenced in accumulo.root tablets section
+        Volume : hdfs://namenode.example.com:8020/accumulo
+Listing volumes referenced in accumulo.root deletes section (volume replacement occurrs at
deletion time)
+
+Listing volumes referenced in accumulo.metadata tablets section
+        Volume : hdfs://namenode.example.com:8020/accumulo
+
+Listing volumes referenced in accumulo.metadata deletes section (volume replacement occurrs
at deletion time)
+\end{verbatim}
+
+After verifying the current volume is correct, shut down the cluster and transition HDFS
to the HA nameservice.
+
+Edit \texttt{\$ACCUMULO\_HOME/conf/accumulo-site.xml} to notify accumulo that a volume is
being replaced. First,
+add the new nameservice volume to the \texttt{instance.volumes} property. Next, add the
+\texttt{instance.volumes.replacements} property in the form of \texttt{old new}. It's important
to not include
+the volume that's being replaced in \texttt{instance.volumes}, otherwise it's possible accumulo
could continue
+to write to the volume.
+
+\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
+<!-- instance.dfs.uri and instance.dfs.dir should not be set-->
+<property>
+  <name>instance.volumes</name>
+  <value>hdfs://nameservice1/accumulo</value>
+</property>
+<property>
+  <name>instance.volumes.replacements</name>
+  <value>hdfs://namenode.example.com:8020/accumulo hdfs://nameservice1/accumulo</value>
+</property>
+\end{verbatim}
+
+Run \texttt{\$ACCUMULO\_HOME/bin/accumulo init --add-volumes} and start up the accumulo cluster.
Verify that the
+new nameservice volume shows up with \texttt{\$ACCUMULO\_HOME/bin/accumulo admin volumes}.
+
+\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
+Listing volumes referenced in zookeeper
+        Volume : hdfs://namenode.example.com:8020/accumulo
+        Volume : hdfs://nameservice1/accumulo
+
+Listing volumes referenced in accumulo.root tablets section
+        Volume : hdfs://namenode.example.com:8020/accumulo
+        Volume : hdfs://nameservice1/accumulo
+Listing volumes referenced in accumulo.root deletes section (volume replacement occurrs at
deletion time)
+
+Listing volumes referenced in accumulo.metadata tablets section
+        Volume : hdfs://namenode.example.com:8020/accumulo
+        Volume : hdfs://nameservice1/accumulo
+Listing volumes referenced in accumulo.metadata deletes section (volume replacement occurrs
at deletion time)
+\end{verbatim}
+
+Some erroneous GarbageCollector messages may still be seen for a small period while data
is transitioning to
+the new volumes. This is expected and can usually be ignored.
+
+

http://git-wip-us.apache.org/repos/asf/accumulo/blob/21ab0557/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 51fa47e..2b654ca 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -158,6 +158,7 @@ public class ServerConstants {
       List<Pair<Path,Path>> ret = new ArrayList<Pair<Path,Path>>();
 
       for (String pair : pairs) {
+
         String uris[] = pair.split("\\s+");
         if (uris.length != 2)
           throw new IllegalArgumentException(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey()
+ " contains malformed pair " + pair);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/21ab0557/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index e0a3797..fd46ab7 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -20,7 +20,6 @@ import static com.google.common.base.Charsets.UTF_8;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN;
-
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Arrays;
@@ -31,7 +30,6 @@ import java.util.Map.Entry;
 import java.util.UUID;
 
 import jline.console.ConsoleReader;
-
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.cli.Help;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -53,6 +51,7 @@ import org.apache.accumulo.core.metadata.RootTable;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.util.Pair;
 import org.apache.accumulo.core.volume.VolumeConfiguration;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
 import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
@@ -122,6 +121,7 @@ public class Initialize {
   }
 
   private static HashMap<String,String> initialMetadataConf = new HashMap<String,String>();
+
   static {
     initialMetadataConf.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "32K");
     initialMetadataConf.put(Property.TABLE_FILE_REPLICATION.getKey(), "5");
@@ -518,17 +518,26 @@ public class Initialize {
   }
 
   private static void addVolumes(VolumeManager fs) throws IOException {
+
+    String[] volumeURIs = VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration());
+
     HashSet<String> initializedDirs = new HashSet<String>();
-    initializedDirs.addAll(Arrays.asList(ServerConstants.checkBaseUris(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()),
true)));
+    initializedDirs.addAll(Arrays.asList(ServerConstants.checkBaseUris(volumeURIs, true)));
 
     HashSet<String> uinitializedDirs = new HashSet<String>();
-    uinitializedDirs.addAll(Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())));
+    uinitializedDirs.addAll(Arrays.asList(volumeURIs));
     uinitializedDirs.removeAll(initializedDirs);
 
     Path aBasePath = new Path(initializedDirs.iterator().next());
     Path iidPath = new Path(aBasePath, ServerConstants.INSTANCE_ID_DIR);
     Path versionPath = new Path(aBasePath, ServerConstants.VERSION_DIR);
 
+    for (Pair<Path,Path> replacementVolume : ServerConstants.getVolumeReplacements())
{
+      if (aBasePath.equals(replacementVolume.getSecond()))
+        log.error(aBasePath + " is set to be replaced in " + Property.INSTANCE_VOLUMES_REPLACEMENTS
+ " and should not appear in " +
+            Property.INSTANCE_VOLUMES + ". It is highly recommended that this property be
removed as data could still be written to this volume.");
+    }
+
     UUID uuid = UUID.fromString(ZooUtil.getInstanceIDFromHdfs(iidPath, ServerConfiguration.getSiteConfiguration()));
 
     if (ServerConstants.DATA_VERSION != Accumulo.getAccumuloPersistentVersion(versionPath.getFileSystem(CachedConfiguration.getInstance()),
versionPath)) {


Mime
View raw message