ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alejan...@apache.org
Subject ambari git commit: AMBARI-9990. CopyFromLocal failed to copy Tez tarball to HDFS failed because multiple processes tried to copy to the same destination simultaneously (alejandro)
Date Wed, 11 Mar 2015 17:15:27 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.0.0 da5dee4c7 -> fa1f08f7a


AMBARI-9990. CopyFromLocal failed to copy Tez tarball to HDFS failed because multiple processes
tried to copy to the same destination simultaneously (alejandro)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fa1f08f7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fa1f08f7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fa1f08f7

Branch: refs/heads/branch-2.0.0
Commit: fa1f08f7a96820bcdf1be24fa9282491965962cc
Parents: da5dee4
Author: Alejandro Fernandez <afernandez@hortonworks.com>
Authored: Wed Mar 11 10:14:51 2015 -0700
Committer: Alejandro Fernandez <afernandez@hortonworks.com>
Committed: Wed Mar 11 10:14:51 2015 -0700

----------------------------------------------------------------------
 .../functions/dynamic_variable_interpretation.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/fa1f08f7/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py
b/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py
index 00b8d70..1e70219 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py
@@ -23,6 +23,7 @@ import os
 import glob
 import re
 import tempfile
+import uuid
 from resource_management.libraries.functions.default import default
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.resources.copy_from_local import CopyFromLocal
@@ -101,17 +102,35 @@ def _copy_files(source_and_dest_pairs, component_user, file_owner, group_owner,
                              mode=0555
         )
 
+        # Because CopyFromLocal does not guarantee synchronization, it's possible for two
processes to first attempt to
+        # copy the file to a temporary location, then process 2 fails because the temporary
file was already created by
+        # process 1, so process 2 tries to clean up by deleting the temporary file, and then
process 1
+        # cannot finish the copy to the final destination, and both fail!
+        # For this reason, the file name on the destination must be unique, and we then rename
it to the intended value.
+        # The rename operation is synchronized by the Namenode.
+        orig_dest_file_name = os.path.split(destination)[1]
+        unique_string = str(uuid.uuid4())[:8]
+        new_dest_file_name = orig_dest_file_name + "." + unique_string
+        new_destination = os.path.join(destination_dir, new_dest_file_name)
         CopyFromLocal(source,
                       mode=0444,
                       owner=file_owner,
                       group=group_owner,
                       user=params.hdfs_user,               # this will be the user to run
the commands as
                       dest_dir=destination_dir,
+                      dest_file=new_dest_file_name,
                       kinnit_if_needed=kinit_if_needed,
                       hdfs_user=params.hdfs_user,
                       hadoop_bin_dir=params.hadoop_bin_dir,
                       hadoop_conf_dir=params.hadoop_conf_dir
         )
+
+        mv_command = format("fs -mv {new_destination} {destination}")
+        ExecuteHadoop(mv_command,
+                      user=params.hdfs_user,
+                      bin_dir=params.hadoop_bin_dir,
+                      conf_dir=params.hadoop_conf_dir
+        )
       except Exception, e:
         Logger.error("Failed to copy file. Source: %s, Destination: %s. Error: %s" % (source,
destination, e.message))
         return_value = 1


Mime
View raw message