ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [01/25] ambari git commit: AMBARI-20175. Rebalance HDFS operation returns after the command is issued. (Laszlo Puskas via stoader)
Date Mon, 06 Mar 2017 18:44:14 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-feature-AMBARI-12556 4527aa15d -> 353cd9dfd


AMBARI-20175. Rebalance HDFS operation returns after the command is issued. (Laszlo Puskas
via stoader)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f896b841
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f896b841
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f896b841

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: f896b84116573c5432b60bdf90315edad20e61de
Parents: a28e0ca
Author: Laszlo Puskas <lpuskas@hortonworks.com>
Authored: Fri Mar 3 20:50:46 2017 +0100
Committer: Toader, Sebastian <stoader@hortonworks.com>
Committed: Fri Mar 3 20:51:16 2017 +0100

----------------------------------------------------------------------
 .../2.1.0.2.0/package/scripts/hdfs_rebalance.py | 32 +++++++++++++++++++-
 .../HDFS/2.1.0.2.0/package/scripts/namenode.py  | 14 ++++++---
 .../3.0.0.3.0/package/scripts/hdfs_rebalance.py | 28 ++++++++++++++++-
 .../HDFS/3.0.0.3.0/package/scripts/namenode.py  | 14 ++++++---
 .../python/stacks/2.0.6/HDFS/test_namenode.py   | 19 ++++++++----
 5 files changed, 91 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/f896b841/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
index 1dc545e..68c609f 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
@@ -19,6 +19,10 @@ limitations under the License.
 '''
 
 import re
+import sys
+from resource_management.core.exceptions import Fail
+from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop
+
 
 class HdfsParser():
   def __init__(self):
@@ -127,4 +131,30 @@ class HdfsLine():
             'bytesBeingMoved': self.bytesBeingMoved,
           }
   def __str__(self):
-    return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, bytesBeingMoved=%d]"%(self.date,
self.iteration, self.bytesAlreadyMoved, self.bytesLeftToMove, self.bytesBeingMoved)
\ No newline at end of file
+    return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, bytesBeingMoved=%d]"%(self.date,
self.iteration, self.bytesAlreadyMoved, self.bytesLeftToMove, self.bytesBeingMoved)
+
+
+
+def is_balancer_running():
+  import params
+  check_balancer_command = "fs -test -e /system/balancer.id"
+  does_hdfs_file_exist = False
+  try:
+    _print("Checking if the balancer is running ...")
+    ExecuteHadoop(check_balancer_command,
+                  user=params.hdfs_user,
+                  logoutput=True,
+                  conf_dir=params.hadoop_conf_dir,
+                  bin_dir=params.hadoop_bin_dir)
+
+    does_hdfs_file_exist = True
+    _print("Balancer is running. ")
+  except Fail:
+    pass
+
+  return does_hdfs_file_exist
+
+def _print(line):
+  sys.stdout.write(line)
+  sys.stdout.flush()
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/f896b841/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
index bd05a95..1347f37 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
@@ -341,10 +341,16 @@ class NameNodeDefault(NameNode):
         self.put_structured_out({'completePercent' : 1})
         return
 
-    Execute(command,
-            on_new_line = handle_new_line,
-            logoutput = False,
-    )
+    if (not hdfs_rebalance.is_balancer_running()):
+      # As the rebalance may take a long time (haours, days) the process is triggered only
+      # Tracking the progress based on the command output is no longer supported due to this
+      Execute(command, wait_for_finish=False)
+
+      _print("The rebalance process has been triggered")
+    else:
+      _print("There is another balancer running. This means you or another Ambari user may
have triggered the "
+             "operation earlier. The process may take a long time to finish (hours, even
days). If the problem persists "
+             "please consult with the HDFS administrators if they have triggred or killed
the operation.")
 
     if params.security_enabled:
       # Delete the kerberos credentials cache (ccache) file

http://git-wip-us.apache.org/repos/asf/ambari/blob/f896b841/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_rebalance.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_rebalance.py
b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_rebalance.py
index 1dc545e..b9b18ef 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_rebalance.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_rebalance.py
@@ -19,6 +19,9 @@ limitations under the License.
 '''
 
 import re
+import sys
+from resource_management.core.exceptions import Fail
+from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop
 
 class HdfsParser():
   def __init__(self):
@@ -127,4 +130,27 @@ class HdfsLine():
             'bytesBeingMoved': self.bytesBeingMoved,
           }
   def __str__(self):
-    return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, bytesBeingMoved=%d]"%(self.date,
self.iteration, self.bytesAlreadyMoved, self.bytesLeftToMove, self.bytesBeingMoved)
\ No newline at end of file
+    return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, bytesBeingMoved=%d]"%(self.date,
self.iteration, self.bytesAlreadyMoved, self.bytesLeftToMove, self.bytesBeingMoved)
+
+def is_balancer_running():
+  import params
+  check_balancer_command = "fs -test -e /system/balancer.id"
+  does_hdfs_file_exist = False
+  try:
+    _print("Checking if the balancer is running ...")
+    ExecuteHadoop(check_balancer_command,
+                  user=params.hdfs_user,
+                  logoutput=True,
+                  conf_dir=params.hadoop_conf_dir,
+                  bin_dir=params.hadoop_bin_dir)
+
+    does_hdfs_file_exist = True
+    _print("Balancer is running. ")
+  except Fail:
+    pass
+
+  return does_hdfs_file_exist
+
+def _print(line):
+  sys.stdout.write(line)
+  sys.stdout.flush()

http://git-wip-us.apache.org/repos/asf/ambari/blob/f896b841/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/namenode.py
b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/namenode.py
index 08578bc..602dad7 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/namenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/namenode.py
@@ -341,10 +341,16 @@ class NameNodeDefault(NameNode):
         self.put_structured_out({'completePercent' : 1})
         return
 
-    Execute(command,
-            on_new_line = handle_new_line,
-            logoutput = False,
-    )
+    if (not hdfs_rebalance.is_balancer_running()):
+      # As the rebalance may take a long time (haours, days) the process is triggered only
+      # Tracking the progress based on the command output is no longer supported due to this
+      Execute(command, wait_for_finish=False)
+
+      _print("The rebalance process has been triggered")
+    else:
+      _print("There is another balancer running. This means you or another Ambari user may
have triggered the "
+             "operation earlier. The process may take a long time to finish (hours, even
days). If the problem persists "
+             "please consult with the HDFS administrators if they have triggred or killed
the operation.")
 
     if params.security_enabled:
       # Delete the kerberos credentials cache (ccache) file

http://git-wip-us.apache.org/repos/asf/ambari/blob/f896b841/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
index ec19dc4..a6a474a 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
@@ -1205,8 +1205,10 @@ class TestNamenode(RMFTestCase):
                               cd_access='a'
                               )
 
+  @patch("hdfs_rebalance.is_balancer_running")
   @patch("resource_management.libraries.script.Script.put_structured_out")
-  def test_rebalance_hdfs(self, pso):
+  def test_rebalance_hdfs(self, pso, hdfs_rebalance_mock):
+      hdfs_rebalance_mock.return_value = False
       self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
                          classname = "NameNode",
                          command = "rebalancehdfs",
@@ -1214,17 +1216,20 @@ class TestNamenode(RMFTestCase):
                          stack_version = self.STACK_VERSION,
                          target = RMFTestCase.TARGET_COMMON_SERVICES
       )
+
       self.assertResourceCalled('Execute', "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export
 PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf balancer -threshold -1'",
-          logoutput = False,
-          on_new_line = FunctionMock('handle_new_line'),
+                                wait_for_finish=False
       )
+
       self.assertNoMoreResources()
 
+  @patch("hdfs_rebalance.is_balancer_running")
   @patch("resource_management.libraries.script.Script.put_structured_out")
   @patch("os.system")
-  def test_rebalance_secured_hdfs(self, pso, system_mock):
+  def test_rebalance_secured_hdfs(self, pso, system_mock, hdfs_rebalance_mock):
 
     system_mock.return_value = -1
+    hdfs_rebalance_mock.return_value = False
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
                        classname = "NameNode",
                        command = "rebalancehdfs",
@@ -1241,13 +1246,15 @@ class TestNamenode(RMFTestCase):
     self.assertResourceCalled('Execute', kinit_cmd,
                               user = 'hdfs',
                               )
+
     self.assertResourceCalled('Execute', rebalance_cmd,
-                              logoutput = False,
-                              on_new_line = FunctionMock('handle_new_line'),
+                              wait_for_finish=False
                               )
+
     self.assertResourceCalled('File', ccache_path,
                               action = ['delete'],
                               )
+
     self.assertNoMoreResources()
 
   @patch("os.path.isfile")


Mime
View raw message