accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [2/3] git commit: ACCUMULO-1832 class for re-writting uris
Date Tue, 05 Nov 2013 18:44:13 GMT
ACCUMULO-1832 class for re-writting uris


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5d7f6a94
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5d7f6a94
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5d7f6a94

Branch: refs/heads/master
Commit: 5d7f6a940e7866a3b79fd563b7008ad5faec4f85
Parents: e86a8d0
Author: Eric Newton <eric.newton@gmail.com>
Authored: Tue Nov 5 13:43:42 2013 -0500
Committer: Eric Newton <eric.newton@gmail.com>
Committed: Tue Nov 5 13:43:42 2013 -0500

----------------------------------------------------------------------
 server/base/pom.xml                             |  10 +-
 .../apache/accumulo/utils/NamespaceRename.java  | 211 +++++++++++++++++++
 2 files changed, 216 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/5d7f6a94/server/base/pom.xml
----------------------------------------------------------------------
diff --git a/server/base/pom.xml b/server/base/pom.xml
index 6f5f4a7..1e3164f 100644
--- a/server/base/pom.xml
+++ b/server/base/pom.xml
@@ -100,13 +100,13 @@
       <artifactId>jetty</artifactId>
     </dependency>
     <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>

http://git-wip-us.apache.org/repos/asf/accumulo/blob/5d7f6a94/server/extras/src/main/java/org/apache/accumulo/utils/NamespaceRename.java
----------------------------------------------------------------------
diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/NamespaceRename.java b/server/extras/src/main/java/org/apache/accumulo/utils/NamespaceRename.java
new file mode 100644
index 0000000..bc0c834
--- /dev/null
+++ b/server/extras/src/main/java/org/apache/accumulo/utils/NamespaceRename.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.utils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.thrift.MasterGoalState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+
+public class NamespaceRename {
+  
+  static final Logger log = LoggerFactory.getLogger(NamespaceRename.class);
+  
+  static class Opts {
+    @Parameter(names = {"--old", "-o"}, required = true)
+    String oldName = null;
+    @Parameter(names = {"--new", "-n"}, required = true)
+    String newName = null;
+  }
+  
+  /**
+   * Utility to recovery from a name node restoration at a new location. For example, if
you had been using "nn1" and the machine died but you were able to
+   * restore the service on a different machine, "nn2" you could rewrite the metadata using
+   * <pre>
+   * accumulo org.apache.accumulo.server.util.NamespaceRename --old hdfs://nn1:9001 --new
hdfs://nn2:9001
+   * </pre>
+   * @param args
+   * @throws Exception 
+   */
+  public static void main(String[] args) throws Exception {
+    Opts opts = new Opts();
+    JCommander cmdline = new JCommander(opts);
+    cmdline.parse(args);
+    log.info("Checking current configuration");
+    AccumuloConfiguration configuration = ServerConfiguration.getSiteConfiguration();
+    checkConfiguration(opts, configuration);
+    Instance instance = HdfsZooInstance.getInstance();
+    log.info("Waiting for HDFS and Zookeeper to be ready");
+    VolumeManager fs = VolumeManagerImpl.get();
+    Accumulo.waitForZookeeperAndHdfs(fs);
+    log.info("Putting servers in SAFE_MODE");
+    ZooReaderWriter zoo = ZooReaderWriter.getInstance();
+    zoo.putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE,
MasterGoalState.SAFE_MODE.toString().getBytes(), NodeExistsPolicy.OVERWRITE);
+    log.info("Updating root table write-ahead logs");
+    updateZookeeper(opts, instance, zoo);
+    log.info("Updating file references in the root table");
+    updateMetaTable(opts, instance, RootTable.NAME);
+    log.info("Updating file references in the metadata table");
+    updateMetaTable(opts, instance, MetadataTable.NAME);
+    zoo.putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE,
MasterGoalState.NORMAL.toString().getBytes(), NodeExistsPolicy.OVERWRITE);
+    log.info("Namespace " + opts.oldName + " has been renamed " + opts.newName);
+  }
+  
+  static final ColumnFQ DIRECTORY_COLUMN = MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
+
+  private static void updateMetaTable(Opts opts, Instance instance, String tableName) throws
Exception,
+      MutationsRejectedException {
+    log.info("Waiting for " + tableName + " to come online");
+    Connector conn = getConnector(instance);
+    Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
+    scanner.fetchColumnFamily(MetadataSchema.TabletsSection.LogColumnFamily.NAME);
+    scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+    DIRECTORY_COLUMN.fetch(scanner);
+    scanner.iterator().hasNext();
+    log.info(tableName + " is online");
+    BatchWriter bw = conn.createBatchWriter(tableName,  new BatchWriterConfig());
+    for (Entry<Key,Value> entry : scanner) {
+      Key key = entry.getKey();
+      Mutation m = new Mutation(key.getRow());
+      if (DIRECTORY_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier())) {
+        m.putDelete(key.getColumnFamily(), key.getColumnQualifier(), key.getTimestamp());
+        String newName = rename(entry.getValue().toString(), opts);
+        m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newName.getBytes()));
+        bw.addMutation(m);
+      } else if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME))
{
+        m.putDelete(key.getColumnFamily(), key.getColumnQualifier(), key.getTimestamp());
+        String newName = rename(key.getColumnQualifier().toString(), opts);
+        m.put(key.getColumnFamily(), new Text(newName), entry.getValue());
+        bw.addMutation(m);
+      } else if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.LogColumnFamily.NAME))
{
+        m.putDelete(key.getColumnFamily(), key.getColumnQualifier(), key.getTimestamp());
+        LogEntry update = convert(LogEntry.fromKeyValue(entry.getKey(), entry.getValue()),
opts);
+        m.put(update.getColumnFamily(), update.getColumnQualifier(), update.getValue());
+        bw.addMutation(m);
+      }
+    }
+    bw.close();
+  }
+
+  static private LogEntry convert(LogEntry entry, Opts opts) {
+    entry.filename = rename(entry.filename, opts);
+    List<String> logSet = new ArrayList<String>();
+    for (String log : entry.logSet) {
+      logSet.add(rename(log, opts));
+    }
+    entry.logSet = logSet;
+    return entry;
+  }
+  
+  private static Connector getConnector(Instance instance) throws AccumuloException, AccumuloSecurityException
{
+    return instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
+  }
+
+  private static void updateZookeeper(Opts opts, Instance instance, ZooReaderWriter zoo)
throws KeeperException, InterruptedException,
+      IOException {
+    String root = ZooUtil.getRoot(instance);
+    String rootTabletLocation = root + RootTable.ZROOT_TABLET_WALOGS;
+    for (String walogName : zoo.getChildren(rootTabletLocation)) {
+      LogEntry entry = new LogEntry();
+      String logZPath = rootTabletLocation + "/" + walogName;
+      byte[] data = zoo.getData(logZPath, null);
+      entry.fromBytes(data);
+      entry = convert(entry, opts);
+      zoo.putPersistentData(logZPath, entry.toBytes(), NodeExistsPolicy.OVERWRITE);
+    }
+    String dirPath = root + RootTable.ZROOT_TABLET_PATH;
+    byte[] dir = zoo.getData(dirPath, null);
+    String newDir = rename(new String(dir), opts);
+    zoo.putPersistentData(dirPath, newDir.getBytes(), NodeExistsPolicy.OVERWRITE);
+  }
+
+  private static String rename(String filename, Opts opts) {
+    if (filename.startsWith(opts.oldName))
+      return opts.newName + filename.substring(opts.oldName.length(), filename.length());
+    return filename;
+  }
+
+  private static void checkConfiguration(Opts opts, AccumuloConfiguration configuration)
throws IOException {
+    if (opts.oldName.endsWith("/"))
+      throw new RuntimeException(opts.oldName + " ends with a slash, do not include it");
+    if (opts.newName.endsWith("/"))
+      throw new RuntimeException(opts.newName + " ends with a slash, do not include it");
+    String volumes = configuration.get(Property.INSTANCE_VOLUMES);
+    if (volumes != null && !volumes.isEmpty()) {
+      Set<String> volumeSet = new HashSet<String>(Arrays.asList(volumes.split(",")));
+      if (volumeSet.contains(opts.oldName))
+        throw new RuntimeException(Property.INSTANCE_VOLUMES.getKey() + " is set to " + volumes
+ " which still contains the old name " + opts.oldName);
+      if (!volumeSet.contains(opts.newName))
+        throw new RuntimeException(Property.INSTANCE_VOLUMES.getKey() + " is set to " + volumes
+ " which does not contain the new name " + opts.oldName);
+      return;
+    } else {
+      String uri = configuration.get(Property.INSTANCE_DFS_URI);
+      if (uri != null && !uri.isEmpty()) {
+        if (!uri.startsWith(opts.newName))
+          throw new RuntimeException(Property.INSTANCE_DFS_DIR.getKey() + " is set to " +
uri + " which is not in " + opts.newName);
+        return;
+      }
+    }
+    FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+    if (!fs.getUri().toString().equals(opts.newName))
+      throw new RuntimeException("Default filesystem is " + fs.getUri() + " and the new name
is " + opts.newName + ". Update your hadoop dfs configuration.");
+  }
+  
+}


Mime
View raw message