Return-Path: X-Original-To: apmail-tez-commits-archive@minotaur.apache.org Delivered-To: apmail-tez-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5F21210124 for ; Fri, 16 Jan 2015 22:23:05 +0000 (UTC) Received: (qmail 88079 invoked by uid 500); 16 Jan 2015 22:23:07 -0000 Delivered-To: apmail-tez-commits-archive@tez.apache.org Received: (qmail 88042 invoked by uid 500); 16 Jan 2015 22:23:07 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 88028 invoked by uid 99); 16 Jan 2015 22:23:07 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 16 Jan 2015 22:23:07 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 07F26E03C4; Fri, 16 Jan 2015 22:23:07 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: gopalv@apache.org To: commits@tez.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: tez git commit: TEZ-1949: Whitelist TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH for broadcast edges. (gopalv) Date: Fri, 16 Jan 2015 22:23:07 +0000 (UTC) Repository: tez Updated Branches: refs/heads/master 4974fb235 -> d676ef2ff TEZ-1949: Whitelist TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH for broadcast edges. (gopalv) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/d676ef2f Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/d676ef2f Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/d676ef2f Branch: refs/heads/master Commit: d676ef2ffb06e3a4debbd994a8880840e77180b9 Parents: 4974fb2 Author: Gopal V Authored: Fri Jan 16 14:19:15 2015 -0800 Committer: Gopal V Committed: Fri Jan 16 14:19:15 2015 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../runtime/library/input/UnorderedKVInput.java | 1 + .../conf/TestUnorderedKVInputConfig.java | 54 ++++++++++++++++++++ 3 files changed, 56 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/d676ef2f/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index eec52dc..1b7f19c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -35,6 +35,7 @@ ALL CHANGES: TEZ-1889. Fix test-patch to provide correct findbugs report. TEZ-1313. Setup pre-commit build to test submitted patches. TEZ-1856. Remove LocalOnFileSortedOutput, LocalMergedInput, LocalTaskOutputFiles. + TEZ-1949. Whitelist TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH for broadcast edges. Release 0.6.0: Unreleased http://git-wip-us.apache.org/repos/asf/tez/blob/d676ef2f/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java index 90e8f0d..cdc0ac0 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java @@ -251,6 +251,7 @@ public class UnorderedKVInput extends AbstractLogicalInput { confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS); confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC); confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH); + confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH); confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT); confKeys.add(TezConfiguration.TEZ_COUNTERS_MAX); confKeys.add(TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH); http://git-wip-us.apache.org/repos/asf/tez/blob/d676ef2f/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java index 5e6e8e9..8c7dea4 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.tez.dag.api.UserPayload; import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; import org.junit.Test; @@ -128,4 +129,57 @@ public class TestUnorderedKVInputConfig { assertEquals("KEY", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, "")); assertEquals("VALUE", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, "")); } + + private final void verifySharedFetchConfigs(UserPayload payload) { + UnorderedKVInputConfig rebuilt = new UnorderedKVInputConfig(); + rebuilt.fromUserPayload(payload); + + Configuration conf = rebuilt.conf; + + assertEquals( + !TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT, + conf.getBoolean( + TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, + TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT)); + + assertEquals( + !TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT, + conf.getBoolean( + TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH, + TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT)); + } + + @Test + public void testSharedFetchConfigs() { + UnorderedKVInputConfig.Builder builder = UnorderedKVInputConfig.newBuilder( + "KEY", "VALUE"); + + UnorderedKVInputConfig configuration = builder + .setAdditionalConfiguration( + TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, + String + .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT)) + .setAdditionalConfiguration( + TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH, + String + .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT)) + .build(); + + verifySharedFetchConfigs(configuration.toUserPayload()); + /* test from configuration */ + + Configuration conf = new Configuration(); + conf.set( + TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, + String + .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT)); + conf.set( + TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH, + String + .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT)); + configuration = UnorderedKVInputConfig.newBuilder("KEY", "VALUE") + .setFromConfiguration(conf).build(); + + verifySharedFetchConfigs(configuration.toUserPayload()); + } }