tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gop...@apache.org
Subject tez git commit: TEZ-1949: Whitelist TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH for broadcast edges. (gopalv)
Date Fri, 16 Jan 2015 23:03:37 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.6 a590516b2 -> b80fe2d1c


TEZ-1949: Whitelist TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH for broadcast edges. (gopalv)

(cherry picked from commit d676ef2ffb06e3a4debbd994a8880840e77180b9)

Conflicts:
	CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b80fe2d1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b80fe2d1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b80fe2d1

Branch: refs/heads/branch-0.6
Commit: b80fe2d1cd7e614bea93f577a11cce989719e5e3
Parents: a590516
Author: Gopal V <gopalv@apache.org>
Authored: Fri Jan 16 15:02:59 2015 -0800
Committer: Gopal V <gopalv@apache.org>
Committed: Fri Jan 16 15:02:59 2015 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../runtime/library/input/UnorderedKVInput.java |  1 +
 .../conf/TestUnorderedKVInputConfig.java        | 54 ++++++++++++++++++++
 3 files changed, 56 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/b80fe2d1/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2707a9b..361c336 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -72,6 +72,7 @@ ALL CHANGES:
   TEZ-1697. DAG submission fails if a local resource added is already part of tez.lib.uris
   TEZ-1060 Add randomness to fault tolerance tests
   TEZ-1790. DeallocationTaskRequest may been handled before corresponding AllocationTaskRequest
in local mode
+  TEZ-1949. Whitelist TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH for broadcast edges.
 
 TEZ-UI CHANGES (TEZ-8):
   TEZ-1799. Enable Cross Origin Support in Tez UI

http://git-wip-us.apache.org/repos/asf/tez/blob/b80fe2d1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
index 0d02cb3..04767ea 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
@@ -249,6 +249,7 @@ public class UnorderedKVInput extends AbstractLogicalInput {
     confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS);
     confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
     confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH);
+    confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH);
     confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT);
     confKeys.add(TezConfiguration.TEZ_COUNTERS_MAX);
     confKeys.add(TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH);

http://git-wip-us.apache.org/repos/asf/tez/blob/b80fe2d1/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
index 5e6e8e9..8c7dea4 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
@@ -29,6 +29,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.dag.api.UserPayload;
 import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
 import org.junit.Test;
 
@@ -128,4 +129,57 @@ public class TestUnorderedKVInputConfig {
     assertEquals("KEY", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, ""));
     assertEquals("VALUE", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, ""));
   }
+
+  private final void verifySharedFetchConfigs(UserPayload payload) {
+    UnorderedKVInputConfig rebuilt = new UnorderedKVInputConfig();
+    rebuilt.fromUserPayload(payload);
+
+    Configuration conf = rebuilt.conf;
+
+    assertEquals(
+        !TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT,
+        conf.getBoolean(
+            TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH,
+            TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT));
+
+    assertEquals(
+        !TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT,
+        conf.getBoolean(
+            TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH,
+            TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT));
+  }
+
+  @Test
+  public void testSharedFetchConfigs() {
+    UnorderedKVInputConfig.Builder builder = UnorderedKVInputConfig.newBuilder(
+        "KEY", "VALUE");
+
+    UnorderedKVInputConfig configuration = builder
+        .setAdditionalConfiguration(
+            TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH,
+            String
+                .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT))
+        .setAdditionalConfiguration(
+            TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH,
+            String
+                .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT))
+        .build();
+
+    verifySharedFetchConfigs(configuration.toUserPayload());
+    /* test from configuration */
+
+    Configuration conf = new Configuration();
+    conf.set(
+        TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH,
+        String
+            .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT));
+    conf.set(
+        TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH,
+        String
+            .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT));
+    configuration = UnorderedKVInputConfig.newBuilder("KEY", "VALUE")
+        .setFromConfiguration(conf).build();
+
+    verifySharedFetchConfigs(configuration.toUserPayload());
+  }
 }


Mime
View raw message