hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From inigo...@apache.org
Subject [02/50] [abbrv] hadoop git commit: MAPREDUCE-6881. Fix warnings from Spotbugs in hadoop-mapreduce. Contributed by Weiwei Yang.
Date Tue, 02 May 2017 22:05:19 GMT
MAPREDUCE-6881. Fix warnings from Spotbugs in hadoop-mapreduce. Contributed by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ed3062f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ed3062f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ed3062f

Branch: refs/heads/HDFS-10467
Commit: 3ed3062fe3979ff55a411b730a8eee2b2c96d6b3
Parents: 28eb2aa
Author: Akira Ajisaka <aajisaka@apache.org>
Authored: Thu Apr 27 15:45:33 2017 +0900
Committer: Akira Ajisaka <aajisaka@apache.org>
Committed: Thu Apr 27 15:45:33 2017 +0900

----------------------------------------------------------------------
 .../hadoop/mapred/LocalContainerLauncher.java   | 46 ++++++++++++--------
 .../hadoop/mapreduce/v2/app/MRAppMaster.java    | 14 +++---
 .../java/org/apache/hadoop/mapred/JVMId.java    |  2 +-
 .../org/apache/hadoop/mapred/Operation.java     | 14 ++++--
 .../mapreduce/v2/hs/HistoryFileManager.java     |  5 ---
 .../org/apache/hadoop/examples/pi/Parser.java   |  8 +++-
 6 files changed, 53 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ed3062f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
index 190d988..0b942b0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
@@ -27,6 +27,8 @@ import java.lang.management.ThreadMXBean;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
+import java.util.Collections;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
@@ -81,7 +83,7 @@ public class LocalContainerLauncher extends AbstractService implements
   private static final Log LOG = LogFactory.getLog(LocalContainerLauncher.class);
 
   private FileContext curFC = null;
-  private final HashSet<File> localizedFiles;
+  private Set<File> localizedFiles = new HashSet<File>();
   private final AppContext context;
   private final TaskUmbilicalProtocol umbilical;
   private final ClassLoader jobClassLoader;
@@ -121,9 +123,12 @@ public class LocalContainerLauncher extends AbstractService implements
     // users who do that get what they deserve (and will have to disable
     // uberization in order to run correctly).
     File[] curLocalFiles = curDir.listFiles();
-    localizedFiles = new HashSet<File>(curLocalFiles.length);
-    for (int j = 0; j < curLocalFiles.length; ++j) {
-      localizedFiles.add(curLocalFiles[j]);
+    if (curLocalFiles != null) {
+      HashSet<File> lf = new HashSet<File>(curLocalFiles.length);
+      for (int j = 0; j < curLocalFiles.length; ++j) {
+        lf.add(curLocalFiles[j]);
+      }
+      localizedFiles = Collections.unmodifiableSet(lf);
     }
 
     // Relocalization note/future FIXME (per chrisdo, 20110315):  At moment,
@@ -521,26 +526,29 @@ public class LocalContainerLauncher extends AbstractService implements
      */
     private void relocalize() {
       File[] curLocalFiles = curDir.listFiles();
-      for (int j = 0; j < curLocalFiles.length; ++j) {
-        if (!localizedFiles.contains(curLocalFiles[j])) {
-          // found one that wasn't there before:  delete it
-          boolean deleted = false;
-          try {
-            if (curFC != null) {
-              // this is recursive, unlike File delete():
-              deleted = curFC.delete(new Path(curLocalFiles[j].getName()),true);
+      if (curLocalFiles != null) {
+        for (int j = 0; j < curLocalFiles.length; ++j) {
+          if (!localizedFiles.contains(curLocalFiles[j])) {
+            // found one that wasn't there before:  delete it
+            boolean deleted = false;
+            try {
+              if (curFC != null) {
+                // this is recursive, unlike File delete():
+                deleted =
+                    curFC.delete(new Path(curLocalFiles[j].getName()), true);
+              }
+            } catch (IOException e) {
+              deleted = false;
+            }
+            if (!deleted) {
+              LOG.warn("Unable to delete unexpected local file/dir "
+                  + curLocalFiles[j].getName()
+                  + ": insufficient permissions?");
             }
-          } catch (IOException e) {
-            deleted = false;
-          }
-          if (!deleted) {
-            LOG.warn("Unable to delete unexpected local file/dir "
-                + curLocalFiles[j].getName() + ": insufficient permissions?");
           }
         }
       }
     }
-
   } // end EventHandler
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ed3062f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index bb7abb6..ba7f929 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -572,13 +572,15 @@ public class MRAppMaster extends CompositeService {
   private boolean isJobNamePatternMatch(JobConf conf, String jobTempDir) {
     // Matched staging files should be preserved after job is finished.
     if (conf.getKeepTaskFilesPattern() != null && jobTempDir != null) {
-      String jobFileName = Paths.get(jobTempDir).getFileName().toString();
-      Pattern pattern = Pattern.compile(conf.getKeepTaskFilesPattern());
-      Matcher matcher = pattern.matcher(jobFileName);
-      return matcher.find();
-    } else {
-      return false;
+      java.nio.file.Path pathName = Paths.get(jobTempDir).getFileName();
+      if (pathName != null) {
+        String jobFileName = pathName.toString();
+        Pattern pattern = Pattern.compile(conf.getKeepTaskFilesPattern());
+        Matcher matcher = pattern.matcher(jobFileName);
+        return matcher.find();
+      }
     }
+    return false;
   }
 
   private boolean isKeepFailedTaskFiles(JobConf conf) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ed3062f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java
index 77a7f8a..6e0e1c4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java
@@ -98,7 +98,7 @@ class JVMId {
     int jobComp = this.jobId.compareTo(that.jobId);
     if(jobComp == 0) {
       if(this.isMap == that.isMap) {
-        return Long.valueOf(this.jvmId).compareTo(that.jvmId);
+        return Long.compare(this.jvmId, that.jvmId);
       } else {
         return this.isMap ? -1 : 1;
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ed3062f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java
index 4df9be4..a5552f8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java
@@ -34,12 +34,20 @@ public enum Operation {
   KILL_TASK(QueueACL.ADMINISTER_JOBS, JobACL.MODIFY_JOB),
   SET_JOB_PRIORITY(QueueACL.ADMINISTER_JOBS, JobACL.MODIFY_JOB),
   SUBMIT_JOB(QueueACL.SUBMIT_JOB, null);
-  
-  public QueueACL qACLNeeded;
-  public JobACL jobACLNeeded;
+
+  private final QueueACL qACLNeeded;
+  private final JobACL jobACLNeeded;
   
   Operation(QueueACL qACL, JobACL jobACL) {
     this.qACLNeeded = qACL;
     this.jobACLNeeded = jobACL;
   }
+
+  public QueueACL getqACLNeeded() {
+    return qACLNeeded;
+  }
+
+  public JobACL getJobACLNeeded() {
+    return jobACLNeeded;
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ed3062f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index bd8d9c2..6b98e47 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -407,17 +407,12 @@ public class HistoryFileManager extends AbstractService {
         }
         JobId jobId = jobIndexInfo.getJobId();
 
-        List<Path> paths = new ArrayList<Path>(2);
         if (historyFile == null) {
           LOG.info("No file for job-history with " + jobId + " found in cache!");
-        } else {
-          paths.add(historyFile);
         }
 
         if (confFile == null) {
           LOG.info("No file for jobConf with " + jobId + " found in cache!");
-        } else {
-          paths.add(confFile);
         }
 
         if (summaryFile == null || !intermediateDoneDirFc.util().exists(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ed3062f/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java
b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java
index a2db9d1..60b3638 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java
@@ -67,8 +67,12 @@ public final class Parser {
   private void parse(File f, Map<Parameter, List<TaskResult>> sums) throws IOException
{
     if (f.isDirectory()) {
       println("Process directory " + f);
-      for(File child : f.listFiles())
-        parse(child, sums);
+      File[] files = f.listFiles();
+      if (files != null) {
+        for(File child : files) {
+          parse(child, sums);
+        }
+      }
     } else if (f.getName().endsWith(".txt")) {
       println("Parse file " + f);
       final Map<Parameter, List<TaskResult>> m = new TreeMap<Parameter, List<TaskResult>>();
   


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message