crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject crunch git commit: CRUNCH-636: amend Make replication factor for temporary files configurable
Date Fri, 24 Mar 2017 05:18:11 GMT
Repository: crunch
Updated Branches:
  refs/heads/master e176b6166 -> ca8ff16b6


CRUNCH-636: amend Make replication factor for temporary files configurable

Signed-off-by: Josh Wills <jwills@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/ca8ff16b
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/ca8ff16b
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/ca8ff16b

Branch: refs/heads/master
Commit: ca8ff16b6172be6cf2be1c185c41593ce4897c53
Parents: e176b61
Author: Attila Sasvari <asasvari@cloudera.com>
Authored: Thu Mar 23 21:35:36 2017 +0100
Committer: Josh Wills <jwills@apache.org>
Committed: Thu Mar 23 16:22:11 2017 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java  | 4 ++--
 .../java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java   | 5 ++---
 2 files changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/ca8ff16b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java
index d31bfad..7705d4a 100644
--- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java
+++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java
@@ -329,8 +329,8 @@ class JobPrototype {
     return d;
   }
 
-  private void setInitialJobReplicationConfig(Configuration job, String defaultReplication)
{
-    job.set(DFS_REPLICATION_INITIAL, defaultReplication);
+  private void setInitialJobReplicationConfig(Configuration jobConf, String defaultReplication)
{
+    jobConf.set(DFS_REPLICATION_INITIAL, defaultReplication);
   }
 
   private static CrunchControlledJob.Hook getHook(

http://git-wip-us.apache.org/repos/asf/crunch/blob/ca8ff16b/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java
b/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java
index 44da01a..e9847c7 100644
--- a/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java
+++ b/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java
@@ -48,7 +48,6 @@ public class JobPrototypeTest {
   @Mock private Path mockPath;
   @Mock private FileTargetImpl mockTarget;
   @Mock private FileSystem mockFs;
-  @Mock private DoNode mockNode;
   @Mock private PGroupedTableImpl<String, String> mockPgroup;
   @Mock private Set<NodePath> mockInputs;
   private JobPrototype jobPrototypeUnderTest;
@@ -86,7 +85,6 @@ public class JobPrototypeTest {
 
   @Test
   public void initialReplicationFactorUsedFromFileSystem() throws IOException {
-    testConfiguration = new Configuration();
     HashMultimap<Target, NodePath> targetNodePaths = HashMultimap.create();
     targetNodePaths.put(mockTarget, new NodePath());
     doReturn(mockPath).when(mockTarget).getPath();
@@ -95,6 +93,7 @@ public class JobPrototypeTest {
     c.set("dfs.replication", TEST_INITIAL_DFS_REPLICATION);
     doReturn(c).when(mockFs).getConf();
     jobPrototypeUnderTest.addReducePaths(targetNodePaths);
+    testConfiguration = new Configuration(false);
 
     jobPrototypeUnderTest.handleInitialReplication(testConfiguration);
     assertEquals(TEST_INITIAL_DFS_REPLICATION, testConfiguration.get("dfs.replication.initial"));
@@ -102,7 +101,6 @@ public class JobPrototypeTest {
 
   @Test
   public void initialReplicationFactorUsedWhenItCannotBeRetrievedFromFileSystem() throws
IOException {
-    testConfiguration = new Configuration();
     HashMultimap<Target, NodePath> targetNodePaths = HashMultimap.create();
     targetNodePaths.put(mockTarget, new NodePath());
     doReturn(mockPath).when(mockTarget).getPath();
@@ -110,6 +108,7 @@ public class JobPrototypeTest {
     Configuration c = new Configuration();
     c.set("dfs.replication", TEST_INITIAL_DFS_REPLICATION);
     jobPrototypeUnderTest.addReducePaths(targetNodePaths);
+    testConfiguration = new Configuration(false);
 
     jobPrototypeUnderTest.handleInitialReplication(testConfiguration);
     assertEquals("3", testConfiguration.get("dfs.replication.initial"));  //default


Mime
View raw message