tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gop...@apache.org
Subject tez git commit: TEZ-1949: Whitelist TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH for broadcast edges. (gopalv)
Date Fri, 16 Jan 2015 22:23:07 GMT
Repository: tez
Updated Branches:
  refs/heads/master 4974fb235 -> d676ef2ff


TEZ-1949: Whitelist TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH for broadcast edges. (gopalv)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/d676ef2f
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/d676ef2f
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/d676ef2f

Branch: refs/heads/master
Commit: d676ef2ffb06e3a4debbd994a8880840e77180b9
Parents: 4974fb2
Author: Gopal V <gopalv@apache.org>
Authored: Fri Jan 16 14:19:15 2015 -0800
Committer: Gopal V <gopalv@apache.org>
Committed: Fri Jan 16 14:19:15 2015 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../runtime/library/input/UnorderedKVInput.java |  1 +
 .../conf/TestUnorderedKVInputConfig.java        | 54 ++++++++++++++++++++
 3 files changed, 56 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/d676ef2f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index eec52dc..1b7f19c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -35,6 +35,7 @@ ALL CHANGES:
   TEZ-1889. Fix test-patch to provide correct findbugs report.
   TEZ-1313. Setup pre-commit build to test submitted patches.
   TEZ-1856. Remove LocalOnFileSortedOutput, LocalMergedInput, LocalTaskOutputFiles.
+  TEZ-1949. Whitelist TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH for broadcast edges.
 
 Release 0.6.0: Unreleased
 

http://git-wip-us.apache.org/repos/asf/tez/blob/d676ef2f/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
index 90e8f0d..cdc0ac0 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
@@ -251,6 +251,7 @@ public class UnorderedKVInput extends AbstractLogicalInput {
     confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS);
     confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
     confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH);
+    confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH);
     confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT);
     confKeys.add(TezConfiguration.TEZ_COUNTERS_MAX);
     confKeys.add(TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH);

http://git-wip-us.apache.org/repos/asf/tez/blob/d676ef2f/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
index 5e6e8e9..8c7dea4 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
@@ -29,6 +29,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.dag.api.UserPayload;
 import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
 import org.junit.Test;
 
@@ -128,4 +129,57 @@ public class TestUnorderedKVInputConfig {
     assertEquals("KEY", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, ""));
     assertEquals("VALUE", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, ""));
   }
+
+  private final void verifySharedFetchConfigs(UserPayload payload) {
+    UnorderedKVInputConfig rebuilt = new UnorderedKVInputConfig();
+    rebuilt.fromUserPayload(payload);
+
+    Configuration conf = rebuilt.conf;
+
+    assertEquals(
+        !TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT,
+        conf.getBoolean(
+            TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH,
+            TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT));
+
+    assertEquals(
+        !TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT,
+        conf.getBoolean(
+            TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH,
+            TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT));
+  }
+
+  @Test
+  public void testSharedFetchConfigs() {
+    UnorderedKVInputConfig.Builder builder = UnorderedKVInputConfig.newBuilder(
+        "KEY", "VALUE");
+
+    UnorderedKVInputConfig configuration = builder
+        .setAdditionalConfiguration(
+            TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH,
+            String
+                .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT))
+        .setAdditionalConfiguration(
+            TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH,
+            String
+                .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT))
+        .build();
+
+    verifySharedFetchConfigs(configuration.toUserPayload());
+    /* test from configuration */
+
+    Configuration conf = new Configuration();
+    conf.set(
+        TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH,
+        String
+            .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT));
+    conf.set(
+        TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH,
+        String
+            .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT));
+    configuration = UnorderedKVInputConfig.newBuilder("KEY", "VALUE")
+        .setFromConfiguration(conf).build();
+
+    verifySharedFetchConfigs(configuration.toUserPayload());
+  }
 }


Mime
View raw message