hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From epa...@apache.org
Subject [hadoop] branch branch-3.2 updated: YARN-9018. Add functionality to AuxiliaryLocalPathHandler to return all locations to read for a given path. Contributed by Kuhu Shukla (kshukla)
Date Thu, 09 Jan 2020 17:39:14 GMT
This is an automated email from the ASF dual-hosted git repository.

epayne pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
     new 3ba0fd1  YARN-9018. Add functionality to AuxiliaryLocalPathHandler to return all
locations to read for a given path. Contributed by Kuhu Shukla (kshukla)
3ba0fd1 is described below

commit 3ba0fd1e50192336d6a0125768739973972917cd
Author: Eric E Payne <ericp@verizonmedia.com>
AuthorDate: Thu Jan 9 17:18:44 2020 +0000

    YARN-9018. Add functionality to AuxiliaryLocalPathHandler to return all locations to read
for a given path. Contributed by Kuhu Shukla (kshukla)
    
    (cherry picked from commit 93233a7d6e4d6b8098622a1aa830355cc18d9589)
---
 .../test/java/org/apache/hadoop/mapred/TestShuffleHandler.java    | 8 ++++++++
 .../apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java  | 8 ++++++++
 .../hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java   | 4 ++++
 .../server/nodemanager/containermanager/ContainerManagerImpl.java | 5 +++++
 4 files changed, 25 insertions(+)

diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 8017248..6988219 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -173,6 +173,14 @@ public class TestShuffleHandler {
         throws IOException {
       return new Path(ABS_LOG_DIR.getAbsolutePath());
     }
+
+    @Override
+    public Iterable<Path> getAllLocalPathsForRead(String path)
+        throws IOException {
+      ArrayList<Path> paths = new ArrayList<>();
+      paths.add(new Path(ABS_LOG_DIR.getAbsolutePath()));
+      return paths;
+    }
   }
 
   private static class MockShuffleHandler2 extends
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java
index 50feecf..452edce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java
@@ -55,4 +55,12 @@ public interface AuxiliaryLocalPathHandler {
    * @throws IOException if the path creations fails
    */
   Path getLocalPathForWrite(String path, long size) throws IOException;
+
+  /**
+   * Get all paths from the local FS for reading for a given Auxiliary Service.
+   * @param path the requested path
+   * @return the complete path list to the file on a local disk as an Iterable
+   * @throws IOException if the file read encounters a problem
+   */
+  Iterable<Path> getAllLocalPathsForRead(String path) throws IOException;
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
index ac0b84d..9af1645 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
@@ -624,6 +624,10 @@ public class LocalDirsHandlerService extends AbstractService {
     return getPathToRead(pathStr, getLocalDirsForRead());
   }
 
+  public Iterable<Path> getAllLocalPathsForRead(String pathStr) throws IOException
{
+    return localDirsAllocator.getAllLocalPathsToRead(pathStr, getConfig());
+  }
+
   public Path getLogPathForWrite(String pathStr, boolean checkWrite)
       throws IOException {
     return logDirsAllocator.getLocalPathForWrite(pathStr,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 8afc52a..0feb3c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -1644,6 +1644,11 @@ public class ContainerManagerImpl extends CompositeService implements
         throws IOException {
       return dirhandlerService.getLocalPathForWrite(path, size, false);
     }
+
+    @Override
+    public Iterable<Path> getAllLocalPathsForRead(String path) throws IOException {
+      return dirhandlerService.getAllLocalPathsForRead(path);
+    }
   }
 
   @SuppressWarnings("unchecked")


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message