ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject [2/3] ambari git commit: AMBARI-18832. Ambari can autoformat NameNode in a production cluster (aonishuk)
Date Thu, 10 Nov 2016 08:54:06 GMT
AMBARI-18832. Ambari can autoformat NameNode in a production cluster (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/30a774f5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/30a774f5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/30a774f5

Branch: refs/heads/branch-2.4
Commit: 30a774f5672ce0426429706e1447bcb4773987fe
Parents: 4146ab9
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Thu Nov 10 10:53:47 2016 +0200
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Thu Nov 10 10:53:47 2016 +0200

----------------------------------------------------------------------
 .../2.1.0.2.0/package/scripts/hdfs_namenode.py  | 22 +++++++++++++-------
 .../python/stacks/2.0.6/HDFS/test_namenode.py   | 10 ++++-----
 2 files changed, 20 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/30a774f5/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
index db52ee1..6e0f5dc 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
@@ -307,7 +307,7 @@ def format_namenode(force=None):
                     conf_dir=hadoop_conf_dir)
     else:
       if not is_namenode_formatted(params):
-        Execute(format("yes Y | hdfs --config {hadoop_conf_dir} namenode -format"),
+        Execute(format("hdfs --config {hadoop_conf_dir} namenode -format -nonInteractive"),
                 user = params.hdfs_user,
                 path = [params.hadoop_bin_dir]
         )
@@ -328,7 +328,7 @@ def format_namenode(force=None):
         nn_name_dirs = params.dfs_name_dir.split(',')
         if not is_namenode_formatted(params):
           try:
-            Execute(format("yes Y | hdfs --config {hadoop_conf_dir} namenode -format"),
+            Execute(format("hdfs --config {hadoop_conf_dir} namenode -format -nonInteractive"),
                     user = params.hdfs_user,
                     path = [params.hadoop_bin_dir]
             )
@@ -384,18 +384,26 @@ def is_namenode_formatted(params):
       )
       marked = True
       
+  if marked:
+    return True
+
   # Check if name dirs are not empty
   for name_dir in nn_name_dirs:
+    code, out = shell.call(("ls", name_dir))
+    dir_exists_and_valid = bool(not code)
+
+    if not dir_exists_and_valid: # situations if disk exists but is crashed at the moment
(ls: reading directory ...: Input/output error)
+      Logger.info(format("NameNode will not be formatted because the directory {name_dir}
is missing or cannot be checked for content. {out}"))
+      return True
+
     try:
       Execute(format("ls {name_dir} | wc -l  | grep -q ^0$"),
       )
-      marked = False
     except Fail:
-      marked = True
-      Logger.info(format("ERROR: Namenode directory(s) is non empty. Will not format the
namenode. List of non-empty namenode dirs {nn_name_dirs}"))
-      break
+      Logger.info(format("NameNode will not be formatted since {name_dir} exists and contains
content"))
+      return True
        
-  return marked
+  return False
 
 @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
 def decommission():

http://git-wip-us.apache.org/repos/asf/ambari/blob/30a774f5/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
index 4cfa2b2..c186af6 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
@@ -57,7 +57,7 @@ class TestNamenode(RMFTestCase):
     )
     self.assert_configure_default()
     self.assertResourceCalled('Execute', 'ls /hadoop/hdfs/namenode | wc -l  | grep -q ^0$',)
-    self.assertResourceCalled('Execute', 'yes Y | hdfs --config /etc/hadoop/conf namenode
-format',
+    self.assertResourceCalled('Execute', 'hdfs --config /etc/hadoop/conf namenode -format
-nonInteractive',
                               path = ['/usr/bin'],
                               user = 'hdfs',
                               )
@@ -172,7 +172,7 @@ class TestNamenode(RMFTestCase):
     )
     self.assert_configure_default()
     self.assertResourceCalled('Execute', 'ls /hadoop/hdfs/namenode | wc -l  | grep -q ^0$',)
-    self.assertResourceCalled('Execute', 'yes Y | hdfs --config /etc/hadoop/conf namenode
-format',
+    self.assertResourceCalled('Execute', 'hdfs --config /etc/hadoop/conf namenode -format
-nonInteractive',
         path = ['/usr/bin'],
         user = 'hdfs',
     )
@@ -300,7 +300,7 @@ class TestNamenode(RMFTestCase):
     )
     self.assert_configure_secured()
     self.assertResourceCalled('Execute', 'ls /hadoop/hdfs/namenode | wc -l  | grep -q ^0$',)
-    self.assertResourceCalled('Execute', 'yes Y | hdfs --config /etc/hadoop/conf namenode
-format',
+    self.assertResourceCalled('Execute', 'hdfs --config /etc/hadoop/conf namenode -format
-nonInteractive',
         path = ['/usr/bin'],
         user = 'hdfs',
     )
@@ -725,7 +725,7 @@ class TestNamenode(RMFTestCase):
 
     # verify that active namenode was formatted
     self.assertResourceCalled('Execute', 'ls /hadoop/hdfs/namenode | wc -l  | grep -q ^0$',)
-    self.assertResourceCalled('Execute', 'yes Y | hdfs --config /etc/hadoop/conf namenode
-format',
+    self.assertResourceCalled('Execute', 'hdfs --config /etc/hadoop/conf namenode -format
-nonInteractive',
         path = ['/usr/bin'],
         user = 'hdfs',
     )
@@ -1775,7 +1775,7 @@ class TestNamenode(RMFTestCase):
       config_dict = json_content,
       stack_version = self.STACK_VERSION,
       target = RMFTestCase.TARGET_COMMON_SERVICES,
-      call_mocks = [(0, None, ''), (0, None)],
+      call_mocks = [(0, None), (0, None, ''), (0, None)],
       mocks_dict = mocks_dict)
 
     # jump right to the start of the NN and then verify that we DO NOT call HdfsResource
after


Mime
View raw message