ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-13241 Post RU dfs_data_dir_mount.hist is lost (dsen)
Date Fri, 25 Sep 2015 17:06:39 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk ad31c9e8d -> 80e2f2036


AMBARI-13241 Post RU dfs_data_dir_mount.hist is lost (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/80e2f203
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/80e2f203
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/80e2f203

Branch: refs/heads/trunk
Commit: 80e2f2036539e5cf55924c61ab56599092c1987d
Parents: ad31c9e
Author: Dmytro Sen <dsen@apache.org>
Authored: Fri Sep 25 20:06:24 2015 +0300
Committer: Dmytro Sen <dsen@apache.org>
Committed: Fri Sep 25 20:06:24 2015 +0300

----------------------------------------------------------------------
 .../resource_management/TestDatanodeHelper.py   | 45 ++------------------
 .../libraries/functions/dfs_datanode_helper.py  | 42 ++++++------------
 .../2.1.0.2.0/package/scripts/hdfs_datanode.py  |  8 +++-
 .../python/stacks/2.0.6/HDFS/test_datanode.py   | 16 +++++++
 4 files changed, 40 insertions(+), 71 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/80e2f203/ambari-agent/src/test/python/resource_management/TestDatanodeHelper.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestDatanodeHelper.py b/ambari-agent/src/test/python/resource_management/TestDatanodeHelper.py
index a74cc0b..70539ac 100644
--- a/ambari-agent/src/test/python/resource_management/TestDatanodeHelper.py
+++ b/ambari-agent/src/test/python/resource_management/TestDatanodeHelper.py
@@ -65,8 +65,7 @@ class TestDatanodeHelper(TestCase):
 
   @patch.object(Logger, "info")
   @patch.object(Logger, "error")
-  @patch.object(dfs_datanode_helper, "_write_data_dir_to_mount_in_file")
-  def test_normalized(self, mock_write_data_dir_to_file, log_error, log_info):
+  def test_normalized(self, log_error, log_info):
     """
     Test that the data dirs are normalized by removing leading and trailing whitespace, and
case sensitive.
     """
@@ -88,41 +87,13 @@ class TestDatanodeHelper(TestCase):
 
     self.assertEquals(0, log_error.call_count)
 
-
-  @patch.object(Logger, "info")
-  @patch.object(Logger, "error")
-  @patch.object(dfs_datanode_helper, "_write_data_dir_to_mount_in_file")
-  @patch.object(dfs_datanode_helper, "get_mount_point_for_dir")
-  @patch.object(os.path, "isdir")
-  def test_save_mount_points(self, mock_os_isdir, mock_get_mount_point, mock_write_data_dir_to_mount_in_file,
log_error, log_info):
-    """
-    Test when all mounts are on root.
-    """
-    mock_get_mount_point.side_effect = ["/", "/", "/"] * 2
-    mock_os_isdir.side_effect = [False, False, False] + [True, True, True]
-    mock_write_data_dir_to_mount_in_file.return_value = True
-
-    # Function under test
-    dfs_datanode_helper.handle_dfs_data_dir(fake_create_dir, self.params, update_cache=False)
-
-    for (name, args, kwargs) in log_info.mock_calls:
-      print args[0]
-
-    for (name, args, kwargs) in log_error.mock_calls:
-      print args[0]
-
-    self.assertEquals(0, log_error.call_count)
-    mock_write_data_dir_to_mount_in_file.assert_called_once_with(self.params, {self.grid0:
"/", self.grid1: "/", self.grid2: "/"})
-
-
   @patch.object(Logger, "info")
   @patch.object(Logger, "error")
   @patch.object(dfs_datanode_helper, "get_data_dir_to_mount_from_file")
-  @patch.object(dfs_datanode_helper, "_write_data_dir_to_mount_in_file")
   @patch.object(dfs_datanode_helper, "get_mount_point_for_dir")
   @patch.object(os.path, "isdir")
   @patch.object(os.path, "exists")
-  def test_grid_becomes_unmounted(self, mock_os_exists, mock_os_isdir, mock_get_mount_point,
mock_write_data_dir_to_mount_in_file, mock_get_data_dir_to_mount_from_file, log_error, log_info):
+  def test_grid_becomes_unmounted(self, mock_os_exists, mock_os_isdir, mock_get_mount_point,
mock_get_data_dir_to_mount_from_file, log_error, log_info):
     """
     Test when grid2 becomes unmounted
     """
@@ -134,7 +105,6 @@ class TestDatanodeHelper(TestCase):
     # Grid2 then becomes unmounted
     mock_get_mount_point.side_effect = ["/dev0", "/dev1", "/"] * 2
     mock_os_isdir.side_effect = [False, False, False] + [True, True, True]
-    mock_write_data_dir_to_mount_in_file.return_value = True
 
     # Function under test
     dfs_datanode_helper.handle_dfs_data_dir(fake_create_dir, self.params, update_cache=False)
@@ -151,18 +121,13 @@ class TestDatanodeHelper(TestCase):
     self.assertEquals(1, log_error.call_count)
     self.assertTrue("Directory /grid/2/data does not exist and became unmounted from /dev2"
in error_msg)
 
-    # Notice that grid2 is still written with its original mount point because an error occurred
on it
-    mock_write_data_dir_to_mount_in_file.assert_called_once_with(self.params, {self.grid0:
"/dev0", self.grid1: "/dev1", self.grid2: "/dev2"})
-
-
   @patch.object(Logger, "info")
   @patch.object(Logger, "error")
   @patch.object(dfs_datanode_helper, "get_data_dir_to_mount_from_file")
-  @patch.object(dfs_datanode_helper, "_write_data_dir_to_mount_in_file")
   @patch.object(dfs_datanode_helper, "get_mount_point_for_dir")
   @patch.object(os.path, "isdir")
   @patch.object(os.path, "exists")
-  def test_grid_becomes_remounted(self, mock_os_exists, mock_os_isdir, mock_get_mount_point,
mock_write_data_dir_to_mount_in_file, mock_get_data_dir_to_mount_from_file, log_error, log_info):
+  def test_grid_becomes_remounted(self, mock_os_exists, mock_os_isdir, mock_get_mount_point,
mock_get_data_dir_to_mount_from_file, log_error, log_info):
     """
     Test when grid2 becomes remounted
     """
@@ -174,7 +139,6 @@ class TestDatanodeHelper(TestCase):
     # Grid2 then becomes remounted
     mock_get_mount_point.side_effect = ["/dev0", "/dev1", "/dev2"] * 2
     mock_os_isdir.side_effect = [False, False, False] + [True, True, True]
-    mock_write_data_dir_to_mount_in_file.return_value = True
 
     # Function under test
     dfs_datanode_helper.handle_dfs_data_dir(fake_create_dir, self.params, update_cache=False)
@@ -186,6 +150,3 @@ class TestDatanodeHelper(TestCase):
       print args[0]
 
     self.assertEquals(0, log_error.call_count)
-
-    # Notice that grid2 is now written with its new mount point to prevent a regression
-    mock_write_data_dir_to_mount_in_file.assert_called_once_with(self.params, {self.grid0:
"/dev0", self.grid1: "/dev1", self.grid2: "/dev2"})
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/80e2f203/ambari-common/src/main/python/resource_management/libraries/functions/dfs_datanode_helper.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/dfs_datanode_helper.py
b/ambari-common/src/main/python/resource_management/libraries/functions/dfs_datanode_helper.py
index 778d869..33e7b41 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/dfs_datanode_helper.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/dfs_datanode_helper.py
@@ -25,31 +25,14 @@ import os
 from resource_management.libraries.functions.file_system import get_mount_point_for_dir,
get_and_cache_mount_points
 from resource_management.core.logger import Logger
 
-
-def _write_data_dir_to_mount_in_file(params, new_data_dir_to_mount_point):
-  """
-  :param new_data_dir_to_mount_point: Dictionary to write to the data_dir_mount_file file,
where
-  the key is each DFS data dir, and the value is its current mount point.
-  :return: Returns True on success, False otherwise.
-  """
-  # Overwrite the existing file, or create it if doesn't exist
-  if params.data_dir_mount_file:
-    try:
-      with open(str(params.data_dir_mount_file), "w") as f:
-        f.write("# This file keeps track of the last known mount-point for each DFS data
dir.\n")
-        f.write("# It is safe to delete, since it will get regenerated the next time that
the DataNode starts.\n")
-        f.write("# However, it is not advised to delete this file since Ambari may \n")
-        f.write("# re-create a DFS data dir that used to be mounted on a drive but is now
mounted on the root.\n")
-        f.write("# Comments begin with a hash (#) symbol\n")
-        f.write("# data_dir,mount_point\n")
-        for kv in new_data_dir_to_mount_point.iteritems():
-          f.write(kv[0] + "," + kv[1] + "\n")
-    except Exception, e:
-      Logger.error("Encountered error while attempting to save DFS data dir mount mount values
to file %s" %
-                   str(params.data_dir_mount_file))
-      return False
-  return True
-
+DATA_DIR_TO_MOUNT_HEADER = """
+# This file keeps track of the last known mount-point for each DFS data dir.
+# It is safe to delete, since it will get regenerated the next time that the DataNode starts.
+# However, it is not advised to delete this file since Ambari may
+# re-create a DFS data dir that used to be mounted on a drive but is now mounted on the root.
+# Comments begin with a hash (#) symbol
+# data_dir,mount_point
+"""
 
 def get_data_dir_to_mount_from_file(params):
   """
@@ -96,6 +79,7 @@ def handle_dfs_data_dir(func, params, update_cache=True):
                will be called as func(data_dir, params)
   :param params: parameters to pass to function pointer
   :param update_cache: Bool indicating whether to update the global cache of mount points
+  :return: Returns a data_dir_mount_file content
   """
 
   # Get the data dirs that Ambari knows about and their last known mount point
@@ -172,9 +156,6 @@ def handle_dfs_data_dir(func, params, update_cache=True):
       curr_mount_point = get_mount_point_for_dir(data_dir)
       data_dir_to_mount_point[data_dir] = curr_mount_point
 
-  # Save back to the file
-  _write_data_dir_to_mount_in_file(params, data_dir_to_mount_point)
-
   if error_messages and len(error_messages) > 0:
     header = " ERROR ".join(["*****"] * 6)
     header = "\n" + "\n".join([header, ] * 3) + "\n"
@@ -183,4 +164,9 @@ def handle_dfs_data_dir(func, params, update_cache=True):
           "root partition, either update the contents of {0}, or delete that file.".format(params.data_dir_mount_file)
     Logger.error(header + msg + header)
 
+  data_dir_to_mount = DATA_DIR_TO_MOUNT_HEADER
+  for kv in data_dir_to_mount_point.iteritems():
+    data_dir_to_mount += kv[0] + "," + kv[1] + "\n"
+
+  return data_dir_to_mount
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/80e2f203/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_datanode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_datanode.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_datanode.py
index df847bd..34ec8cd 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_datanode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_datanode.py
@@ -48,7 +48,13 @@ def datanode(action=None):
               owner=params.hdfs_user,
               group=params.user_group)
 
-    handle_dfs_data_dir(create_dirs, params)
+    File(params.data_dir_mount_file,
+         owner=params.hdfs_user,
+         group=params.user_group,
+         mode=0644,
+         content=handle_dfs_data_dir(create_dirs, params)
+    )
+
   elif action == "start" or action == "stop":
     import params
     service(

http://git-wip-us.apache.org/repos/asf/ambari/blob/80e2f203/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py
index 8e6e386..72925a0 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py
@@ -23,6 +23,8 @@ from mock.mock import MagicMock, patch
 from resource_management.libraries.script import Script
 from resource_management.core import shell
 from resource_management.core.exceptions import Fail
+import resource_management.libraries.functions.dfs_datanode_helper
+
 
 class TestDatanode(RMFTestCase):
   COMMON_SERVICES_PACKAGE_DIR = "HDFS/2.1.0.2.0/package"
@@ -355,6 +357,13 @@ class TestDatanode(RMFTestCase):
                               recursive = True,
                               cd_access='a'
                               )
+    content = resource_management.libraries.functions.dfs_datanode_helper.DATA_DIR_TO_MOUNT_HEADER
+    self.assertResourceCalled('File', '/etc/hadoop/conf/dfs_data_dir_mount.hist',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              mode = 0644,
+                              content = content
+                              )
 
   def assert_configure_secured(self, stackVersion=STACK_VERSION, snappy_enabled=True):
     conf_dir = '/etc/hadoop/conf'
@@ -420,6 +429,13 @@ class TestDatanode(RMFTestCase):
                               recursive = True,
                               cd_access='a'
                               )
+    content = resource_management.libraries.functions.dfs_datanode_helper.DATA_DIR_TO_MOUNT_HEADER
+    self.assertResourceCalled('File', '/etc/hadoop/conf/dfs_data_dir_mount.hist',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              mode = 0644,
+                              content = content
+                              )
 
 
   def test_pre_rolling_restart(self):


Mime
View raw message