tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject tez git commit: TEZ-3559. TEZ_LIB_URIS doesn't work with schemes different than the defaultFS (Eric Badge via jeagles)
Date Thu, 15 Dec 2016 23:43:10 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.7 e849f0981 -> 4a2316ba8


TEZ-3559. TEZ_LIB_URIS doesn't work with schemes different than the defaultFS (Eric Badge
via jeagles)

(cherry picked from commit 4f9b7fdf2049c2a96d6c6cb047abb91c58cf2737)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/4a2316ba
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/4a2316ba
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/4a2316ba

Branch: refs/heads/branch-0.7
Commit: 4a2316ba803e5058f0724a3d5efc16c6c4895679
Parents: e849f09
Author: Jonathan Eagles <jeagles@yahoo-inc.com>
Authored: Thu Dec 15 17:31:12 2016 -0600
Committer: Jonathan Eagles <jeagles@yahoo-inc.com>
Committed: Thu Dec 15 17:42:54 2016 -0600

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/tez/client/TezClientUtils.java   |  5 ++--
 .../apache/tez/client/TestTezClientUtils.java   | 31 ++++++++++++++++++++
 3 files changed, 34 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/4a2316ba/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c652351..7204600 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3559. TEZ_LIB_URIS doesn't work with schemes different than the defaultFS
   TEZ-3549. TaskAttemptImpl does not initialize TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS correctly
   TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768
to Tez
   TEZ-3507. Task logs link when editing url from one task to another.

http://git-wip-us.apache.org/repos/asf/tez/blob/4a2316ba/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 7a5bf7c..67e9979 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -269,7 +269,7 @@ public class TezClientUtils {
         }
 
         LocalResourceVisibility lrVisibility;
-        if (checkAncestorPermissionsForAllUsers(conf, url.getFile(),
+        if (checkAncestorPermissionsForAllUsers(conf, p,
             FsAction.EXECUTE) &&
             fStatus.getPermission().getOtherAction().implies(FsAction.READ)) {
           lrVisibility = LocalResourceVisibility.PUBLIC;
@@ -1004,9 +1004,8 @@ public class TezClientUtils {
         + ( javaOpts != null ? javaOpts : "");
   }
 
-  private static boolean checkAncestorPermissionsForAllUsers(Configuration conf, String uri,
+  private static boolean checkAncestorPermissionsForAllUsers(Configuration conf, Path pathComponent,
                                                              FsAction permission) throws
IOException {
-    Path pathComponent = new Path(uri);
     FileSystem fs = pathComponent.getFileSystem(conf);
 
     if (Shell.WINDOWS && fs instanceof LocalFileSystem) {

http://git-wip-us.apache.org/repos/asf/tez/blob/4a2316ba/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index c630db4..efa6ed4 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -209,6 +209,37 @@ public class TestTezClientUtils {
     assertFalse(localizedMap.isEmpty());
   }
 
+
+    /**
+   *
+   */
+  @Test (timeout=5000)
+  public void testTezDefaultFS() throws Exception {
+    FileSystem localFs = FileSystem.getLocal(new Configuration());
+    StringBuilder tezLibUris = new StringBuilder();
+
+    Path topDir = new Path(TEST_ROOT_DIR, "testTezDefaultFS");
+    if (localFs.exists(topDir)) {
+      localFs.delete(topDir, true);
+    }
+    localFs.mkdirs(topDir);
+
+    Path file = new Path(topDir, "file.jar");
+
+    Assert.assertTrue(localFs.createNewFile(file));
+    tezLibUris.append(localFs.makeQualified(file).toString());
+
+    TezConfiguration conf = new TezConfiguration();
+    conf.set(TezConfiguration.TEZ_LIB_URIS, tezLibUris.toString());
+    conf.set("fs.defaultFS", "hdfs:///localhost:1234");
+    Credentials credentials = new Credentials();
+    Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>();
+    TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap);
+
+    Assert.assertTrue(localFs.delete(file, true));
+    Assert.assertTrue(localFs.delete(topDir, true));
+  }
+
     /**
    *
    */


Mime
View raw message