hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1089324 - in /hadoop/hdfs/branches/yahoo-merge: CHANGES.txt bin/distribute-exclude.sh bin/refresh-namenodes.sh src/java/org/apache/hadoop/hdfs/tools/GetConf.java
Date Wed, 06 Apr 2011 06:22:56 GMT
Author: omalley
Date: Wed Apr  6 06:22:55 2011
New Revision: 1089324

URL: http://svn.apache.org/viewvc?rev=1089324&view=rev
Log:
commit 04eca5fa574f5a4adc4e0482c8120857dedb9352
Author: Tanping Wang <tanping@whenplant.corp.yahoo.com>
Date:   Wed Mar 9 17:26:13 2011 -0800

    HDFS-1811. Create scripts to decomission datanodes. (Erik Steffl)

Added:
    hadoop/hdfs/branches/yahoo-merge/bin/distribute-exclude.sh
    hadoop/hdfs/branches/yahoo-merge/bin/refresh-namenodes.sh
Modified:
    hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/tools/GetConf.java

Modified: hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/CHANGES.txt?rev=1089324&r1=1089323&r2=1089324&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/hdfs/branches/yahoo-merge/CHANGES.txt Wed Apr  6 06:22:55 2011
@@ -209,6 +209,8 @@ Trunk (unreleased changes)
     HDFS-1700. Federation: fsck needs to work with federation changes.
     (Matt Foley via suresh)
 
+    HDFS-1811. Create scripts to decommission datanodes. (Erik Steffl)
+
   IMPROVEMENTS
 
     HDFS-1510. Added test-patch.properties required by test-patch.sh (nigel)

Added: hadoop/hdfs/branches/yahoo-merge/bin/distribute-exclude.sh
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/bin/distribute-exclude.sh?rev=1089324&view=auto
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/bin/distribute-exclude.sh (added)
+++ hadoop/hdfs/branches/yahoo-merge/bin/distribute-exclude.sh Wed Apr  6 06:22:55 2011
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# ------------------------------------------------------------------
+#
+# The purpose of this script is to distribute the exclude file (see
+# "dfs.hosts.exclude" in hdfs-site.xml).
+#
+# Input of the script is a local exclude file. The exclude file
+# will be distributed to all the namenodes. The location on the namenodes
+# is determined by the configuration "dfs.hosts.exclude" in hdfs-site.xml
+# (this value is read from the local copy of hdfs-site.xml and must be same
+# on all the namenodes).
+#
+# The user running this script needs write permissions on the target
+# directory on namenodes.
+#
+# After this command, run refresh-namenodes.sh so that namenodes start
+# using the new exclude file.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin/hdfs-config.sh"
+
+if [ "$1" = '' ] ; then
+  "Error: please specify local exclude file as a first argument"
+  exit 1
+else
+  excludeFilenameLocal=$1
+fi
+
+if [ ! -f "$excludeFilenameLocal" ] ; then
+  echo "Error: exclude file [$excludeFilenameLocal] does not exist."
+  exit 1
+fi
+
+namenodes=$("$HADOOP_HOME/bin/hdfs" getconf -namenodes)
+excludeFilenameRemote=$("$HADOOP_HOME/bin/hdfs" getconf -excludeFile)
+
+if [ "$excludeFilenameRemote" = '' ] ; then
+  echo \
+    "Error: hdfs getconf -excludeFile returned empty string, " \
+    "please setup dfs.hosts.exclude in hdfs-site.xml in local cluster " \
+    "configuration and on all namenodes"
+  exit 1
+fi
+
+echo "Copying exclude file [$excludeFilenameRemote] to namenodes:"
+
+for namenode in $namenodes ; do
+  echo "    [$namenode]"
+  scp "$excludeFilenameLocal" "$namenode:$excludeFilenameRemote"
+  if [ "$?" != '0' ] ; then errorFlag='1' ; fi
+done
+
+if [ "$errorFlag" = '1' ] ; then
+  echo "Error: transfer of exclude file failed, see error messages above."
+  exit 1
+else
+  echo "Transfer of exclude file to all namenodes succeeded."
+fi
+
+# eof

Added: hadoop/hdfs/branches/yahoo-merge/bin/refresh-namenodes.sh
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/bin/refresh-namenodes.sh?rev=1089324&view=auto
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/bin/refresh-namenodes.sh (added)
+++ hadoop/hdfs/branches/yahoo-merge/bin/refresh-namenodes.sh Wed Apr  6 06:22:55 2011
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# ------------------------------------------------------------------
+# This script refreshes all namenodes, it's a simple wrapper
+# for dfsadmin to support multiple namenodes.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin/hdfs-config.sh"
+
+namenodes=$("$HADOOP_HOME/bin/hdfs" getconf -namenodes)
+
+for namenode in $namenodes ; do
+  echo "Refreshing namenode [$namenode]"
+  "$HADOOP_HOME/bin/hdfs" dfsadmin -refreshNodes
+  if [ "$?" != '0' ] ; then errorFlag='1' ; fi
+done
+
+if [ "$errorFlag" = '1' ] ; then
+  echo "Error: refresh of namenodes failed, see error messages above."
+  exit 1
+else
+  echo "Refresh of all namenodes succeeded."
+fi
+
+
+# eof

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/tools/GetConf.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/tools/GetConf.java?rev=1089324&r1=1089323&r2=1089324&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/tools/GetConf.java (original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/tools/GetConf.java Wed
Apr  6 06:22:55 2011
@@ -60,10 +60,12 @@ public class GetConf extends Configured 
         "gets list of secondary namenodes in the cluster."),
     BACKUP("-backupNodes", new BackupNodesCommandHandler(),
         "gets list of backup nodes in the cluster."),
-    INCLUDE_FILE("-includeFile", new CommandHandler("dfs.hosts"),
+    INCLUDE_FILE("-includeFile",
+        new CommandHandler("DFSConfigKeys.DFS_HOSTS"),
         "gets the include file path that defines the datanodes " +
         "that can join the cluster."),
-    EXCLUDE_FILE("-excludeFile", new CommandHandler("dfs.hosts.exlucde"),
+    EXCLUDE_FILE("-excludeFile",
+        new CommandHandler("DFSConfigKeys.DFS_HOSTS_EXCLUDE"),
         "gets the exclude file path that defines the datanodes " +
         "that need to decommissioned.");
 



Mime
View raw message