ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sboi...@apache.org
Subject incubator-ignite git commit: #[IGNITE-218]: fixed by review results.
Date Thu, 04 Jun 2015 13:25:22 GMT
Repository: incubator-ignite
Updated Branches:
  refs/heads/ignite-218 0fa49ed28 -> b347c4b32


#[IGNITE-218]: fixed by review results.


Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/b347c4b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/b347c4b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/b347c4b3

Branch: refs/heads/ignite-218
Commit: b347c4b325df9717bb2d87780a13609109f826db
Parents: 0fa49ed
Author: iveselovskiy <iveselovskiy@gridgain.com>
Authored: Thu Jun 4 16:20:46 2015 +0300
Committer: iveselovskiy <iveselovskiy@gridgain.com>
Committed: Thu Jun 4 16:20:46 2015 +0300

----------------------------------------------------------------------
 .../hadoop/fs/IgniteHadoopFileSystemCounterWriter.java   |  1 -
 .../ignite/internal/processors/hadoop/HadoopUtils.java   | 11 +++++------
 .../internal/processors/hadoop/v2/HadoopV2Job.java       |  4 +---
 .../processors/hadoop/v2/HadoopV2JobResourceManager.java | 11 ++++-------
 4 files changed, 10 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/b347c4b3/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopFileSystemCounterWriter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopFileSystemCounterWriter.java
b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopFileSystemCounterWriter.java
index 2305f1e..d910507 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopFileSystemCounterWriter.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopFileSystemCounterWriter.java
@@ -72,7 +72,6 @@ public class IgniteHadoopFileSystemCounterWriter implements HadoopCounterWriter
         try {
             hadoopCfg.set(MRJobConfig.USER_NAME, user);
 
-            // TODO: Check if FileSystem should be closed, see https://issues.apache.org/jira/browse/IGNITE-980
             FileSystem fs = HadoopUtils.fileSystemForMrUser(jobStatPath.toUri(), hadoopCfg,
true);
 
             fs.mkdirs(jobStatPath);

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/b347c4b3/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
index d419915..68a9ef6 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
@@ -64,7 +64,6 @@ public class HadoopUtils {
     private static final String OLD_REDUCE_CLASS_ATTR = "mapred.reducer.class";
 
     /** Lazy per-user cache for the file systems. It is cleared and nulled in #close() method.
*/
-    // TODO: cleam up this cache upon Ignite node stop, see https://issues.apache.org/jira/browse/IGNITE-980
.
     private static final HadoopLazyConcurrentMap<FsCacheKey, FileSystem> fileSysLazyMap
= new HadoopLazyConcurrentMap<>(
         new HadoopLazyConcurrentMap.ValueFactory<FsCacheKey, FileSystem>() {
             @Override public FileSystem createValue(FsCacheKey key) {
@@ -76,12 +75,14 @@ public class HadoopUtils {
 
                     String scheme = uri.getScheme();
 
+                    // Copy the configuration to avoid altering the external object.
+                    Configuration cfg = new Configuration(key.configuration());
+
                     String prop = HadoopUtils.disableFsCachePropertyName(scheme);
 
-                    // TODO: Copy configuration instead of altering existing one.
-                    key.configuration().setBoolean(prop, true);
+                    cfg.setBoolean(prop, true);
 
-                    return FileSystem.get(uri, key.configuration(), key.user());
+                    return FileSystem.get(uri, cfg, key.user());
                 }
                 catch (IOException | InterruptedException ioe) {
                     throw new IgniteException(ioe);
@@ -166,8 +167,6 @@ public class HadoopUtils {
                 break;
 
             case PHASE_REDUCE:
-                // TODO: temporary fixed, but why PHASE_REDUCE could have 0 reducers?
-                // See https://issues.apache.org/jira/browse/IGNITE-764
                 setupProgress = 1;
                 mapProgress = 1;
 

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/b347c4b3/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
index 849bbe6..d754039 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
@@ -134,7 +134,6 @@ public class HadoopV2Job implements HadoopJob {
             Path jobDir = new Path(jobDirPath);
 
             try {
-                // TODO: Check if FileSystem should be closed, see https://issues.apache.org/jira/browse/IGNITE-980
                 FileSystem fs = fileSystemForMrUser(jobDir.toUri(), jobConf, true);
 
                 JobSplit.TaskSplitMetaInfo[] metaInfos = SplitMetaInfoReader.readSplitMetaInfo(hadoopJobID,
fs, jobConf,
@@ -322,8 +321,7 @@ public class HadoopV2Job implements HadoopJob {
 
     /** {@inheritDoc} */
     @Override public void cleanupStagingDirectory() {
-        if (rsrcMgr != null)
-            rsrcMgr.cleanupStagingDirectory();
+        rsrcMgr.cleanupStagingDirectory();
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/b347c4b3/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
index d34d250..2f64e77 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
@@ -40,6 +40,9 @@ import java.util.*;
  * files are needed to be placed on local files system.
  */
 public class HadoopV2JobResourceManager {
+    /** File type Fs disable caching property name. */
+    private static final String FILE_DISABLE_CACHING_PROPERTY_NAME = HadoopUtils.disableFsCachePropertyName("file");
+
     /** Hadoop job context. */
     private final JobContextImpl ctx;
 
@@ -84,10 +87,7 @@ public class HadoopV2JobResourceManager {
         try {
             cfg.set(HadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP, dir.getAbsolutePath());
 
-            // TODO: Make constant.
-            String prop = HadoopUtils.disableFsCachePropertyName("file");
-
-            if (!cfg.getBoolean(prop, false))
+            if (!cfg.getBoolean(FILE_DISABLE_CACHING_PROPERTY_NAME, false))
                 FileSystem.getLocal(cfg).setWorkingDirectory(new Path(dir.getAbsolutePath()));
         }
         finally {
@@ -115,7 +115,6 @@ public class HadoopV2JobResourceManager {
                 stagingDir = new Path(new URI(mrDir));
 
                 if (download) {
-                    // TODO: Check if FileSystem should be closed, see https://issues.apache.org/jira/browse/IGNITE-980
                     FileSystem fs = HadoopUtils.fileSystemForMrUser(stagingDir.toUri(), cfg,
true);
 
                     if (!fs.exists(stagingDir))
@@ -211,7 +210,6 @@ public class HadoopV2JobResourceManager {
 
             FileSystem dstFs = FileSystem.getLocal(cfg);
 
-            // TODO: Check if FileSystem should be closed, see https://issues.apache.org/jira/browse/IGNITE-980
             FileSystem srcFs = HadoopUtils.fileSystemForMrUser(srcPath.toUri(), cfg, true);
 
             if (extract) {
@@ -295,7 +293,6 @@ public class HadoopV2JobResourceManager {
     public void cleanupStagingDirectory() {
         try {
             if (stagingDir != null)
-                // TODO: Check if FileSystem should be closed, see https://issues.apache.org/jira/browse/IGNITE-980
                 HadoopUtils.fileSystemForMrUser(stagingDir.toUri(), ctx.getJobConf(), true).delete(stagingDir,
true);
         }
         catch (Exception e) {


Mime
View raw message