Return-Path: X-Original-To: apmail-tez-commits-archive@minotaur.apache.org Delivered-To: apmail-tez-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B2892113EC for ; Wed, 16 Jul 2014 22:20:46 +0000 (UTC) Received: (qmail 66550 invoked by uid 500); 16 Jul 2014 22:20:46 -0000 Delivered-To: apmail-tez-commits-archive@tez.apache.org Received: (qmail 66524 invoked by uid 500); 16 Jul 2014 22:20:46 -0000 Mailing-List: contact commits-help@tez.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.incubator.apache.org Delivered-To: mailing list commits@tez.incubator.apache.org Received: (qmail 66515 invoked by uid 99); 16 Jul 2014 22:20:46 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Jul 2014 22:20:46 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Wed, 16 Jul 2014 22:20:45 +0000 Received: (qmail 64090 invoked by uid 99); 16 Jul 2014 22:20:19 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Jul 2014 22:20:19 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 3904B99B4B3; Wed, 16 Jul 2014 22:20:19 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hitesh@apache.org To: commits@tez.incubator.apache.org Date: Wed, 16 Jul 2014 22:20:19 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/2] TEZ-1119. Support display of user payloads in Tez UI. (hitesh) X-Virus-Checked: Checked by ClamAV on apache.org Repository: incubator-tez Updated Branches: refs/heads/master 171da568e -> 36d08fe03 http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestRuntimeUtils.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestRuntimeUtils.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestRuntimeUtils.java deleted file mode 100644 index f374cc0..0000000 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestRuntimeUtils.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.runtime; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.net.URL; -import java.util.Collections; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.tez.common.RuntimeUtils; -import org.apache.tez.dag.api.TezException; -import org.junit.Test; - -public class TestRuntimeUtils { - - @Test - public void testAddResourceToClasspath() throws IOException, TezException { - - String rsrcName = "dummyfile.xml"; - FileSystem localFs = FileSystem.getLocal(new Configuration()); - Path p = new Path(rsrcName); - p = localFs.makeQualified(p); - - localFs.delete(p, false); - - try { - URL loadedUrl = null; - - loadedUrl = Thread.currentThread().getContextClassLoader().getResource(rsrcName); - assertNull(loadedUrl); - - // Add parent to classpath since we're not adding a jar - assertTrue(localFs.createNewFile(p)); - String urlForm = p.toUri().toURL().toString(); - urlForm = urlForm.substring(0, urlForm.lastIndexOf('/') + 1); - URL url = new URL(urlForm); - - RuntimeUtils.addResourcesToClasspath(Collections.singletonList(url)); - - loadedUrl = Thread.currentThread().getContextClassLoader().getResource(rsrcName); - - assertNotNull(loadedUrl); - } finally { - localFs.delete(p, false); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java index e37bbe3..ac88865 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtils; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.runtime.api.Event; @@ -90,7 +91,7 @@ public class ShuffleInputEventHandler { } if (shufflePayload.hasEmptyPartitions()) { try { - byte[] emptyPartitions = TezUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions()); + byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions()); BitSet emptyPartitionsBitSet = TezUtils.fromByteArray(emptyPartitions); if (emptyPartitionsBitSet.get(partitionId)) { LOG.info("Source partition: " + partitionId + " did not generate any data. SrcAttempt: [" http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java index 2d51a7d..4e3e94b 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java @@ -48,6 +48,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.TezUtils; import org.apache.tez.common.counters.TaskCounter; @@ -477,7 +478,7 @@ public class UnorderedPartitionedKVWriter extends BaseUnorderedPartitionedKVWrit } if (emptyPartitions.cardinality() != 0) { // Empty partitions exist - ByteString emptyPartitionsByteString = TezUtils.compressByteArrayToByteString(TezUtils + ByteString emptyPartitionsByteString = TezCommonUtils.compressByteArrayToByteString(TezUtils .toByteArray(emptyPartitions)); payloadBuidler.setEmptyPartitions(emptyPartitionsByteString); } http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java index 8fbac92..c1f5ddb 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java @@ -35,6 +35,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.TezRuntimeFrameworkConfigs; import org.apache.tez.common.TezUtils; @@ -165,7 +166,7 @@ public class OnFileSortedOutput extends AbstractLogicalOutput { } if (emptyPartitions > 0) { ByteString emptyPartitionsBytesString = - TezUtils.compressByteArrayToByteString(TezUtils.toByteArray(emptyPartitionDetails)); + TezCommonUtils.compressByteArrayToByteString(TezUtils.toByteArray(emptyPartitionDetails)); payloadBuilder.setEmptyPartitions(emptyPartitionsBytesString); LOG.info("EmptyPartition bitsetSize=" + emptyPartitionDetails.cardinality() + ", numOutputs=" + getNumPhysicalOutputs() + ", emptyPartitions=" + emptyPartitions http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java index 646ddbd..8cd47c1 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java @@ -31,6 +31,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.TezRuntimeFrameworkConfigs; import org.apache.tez.common.TezUtils; @@ -134,7 +135,7 @@ public class OnFileUnorderedKVOutput extends AbstractLogicalOutput { BitSet emptyPartitions = new BitSet(); emptyPartitions.set(0); ByteString emptyPartitionsBytesString = - TezUtils.compressByteArrayToByteString(TezUtils.toByteArray(emptyPartitions)); + TezCommonUtils.compressByteArrayToByteString(TezUtils.toByteArray(emptyPartitions)); payloadBuilder.setEmptyPartitions(emptyPartitionsBytesString); } if (outputGenerated) { http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java index 628298a..65dba32 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java @@ -26,6 +26,7 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtils; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.runtime.api.Event; @@ -101,7 +102,7 @@ public class ShuffleInputEventHandlerImpl implements ShuffleEventHandler { + stringify(shufflePayload)); if (shufflePayload.hasEmptyPartitions()) { - byte[] emptyPartitions = TezUtils.decompressByteStringToByteArray(shufflePayload + byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload .getEmptyPartitions()); BitSet emptyPartionsBitSet = TezUtils.fromByteArray(emptyPartitions); if (emptyPartionsBitSet.get(srcIndex)) { http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java index f45a7d4..5c3a83a 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java @@ -26,7 +26,7 @@ import java.util.Map; import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration; -import org.apache.tez.common.RuntimeUtils; +import org.apache.tez.common.ReflectionUtils; import org.apache.tez.common.TezUtils; import org.apache.tez.dag.api.EdgeManager; import org.apache.tez.dag.api.EdgeManagerContext; @@ -125,7 +125,7 @@ public class TestShuffleVertexManager { newEdgeManagers.clear(); for (Entry entry : ((Map)invocation.getArguments()[2]).entrySet()) { - EdgeManager edgeManager = RuntimeUtils.createClazzInstance( + EdgeManager edgeManager = ReflectionUtils.createClazzInstance( entry.getValue().getClassName()); final byte[] userPayload = entry.getValue().getUserPayload(); edgeManager.initialize(new EdgeManagerContext() { http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java index f5fd276..9b37c78 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java @@ -57,6 +57,7 @@ import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.DefaultCodec; import org.apache.hadoop.util.DiskChecker.DiskErrorException; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.TezRuntimeFrameworkConfigs; import org.apache.tez.common.TezUtils; @@ -316,7 +317,7 @@ public class TestUnorderedPartitionedKVWriter { BitSet emptyPartitionBits = null; if (partitionsWithData.cardinality() != numPartitions) { assertTrue(eventProto.hasEmptyPartitions()); - byte[] emptyPartitions = TezUtils.decompressByteStringToByteArray(eventProto + byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(eventProto .getEmptyPartitions()); emptyPartitionBits = TezUtils.fromByteArray(emptyPartitions); assertEquals(numPartitions - partitionsWithData.cardinality(), @@ -505,7 +506,7 @@ public class TestUnorderedPartitionedKVWriter { emptyPartitionBits = new BitSet(numPartitions); } else { assertTrue(eventProto.hasEmptyPartitions()); - byte[] emptyPartitions = TezUtils.decompressByteStringToByteArray(eventProto + byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(eventProto .getEmptyPartitions()); emptyPartitionBits = TezUtils.fromByteArray(emptyPartitions); if (numRecordsWritten == 0) { http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java index e95c30d..2aa7396 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java @@ -27,6 +27,7 @@ import java.util.BitSet; import java.util.LinkedList; import java.util.List; +import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtils; import org.apache.tez.runtime.api.Event; import org.apache.tez.runtime.api.TezInputContext; @@ -151,7 +152,7 @@ public class TestShuffleInputEventHandlerImpl { for (int i : emptyPartitions) { bitSet.set(i); } - ByteString emptyPartitionsBytesString = TezUtils.compressByteArrayToByteString(TezUtils + ByteString emptyPartitionsBytesString = TezCommonUtils.compressByteArrayToByteString(TezUtils .toByteArray(bitSet)); return emptyPartitionsBytesString; } http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java ---------------------------------------------------------------------- diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java index e5667c7..ea0378d 100644 --- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java +++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java @@ -67,7 +67,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.tez.client.TezClientUtils; import org.apache.tez.client.TezClient; import org.apache.tez.client.TezAppMasterStatus; -import org.apache.tez.common.RuntimeUtils; +import org.apache.tez.common.ReflectionUtils; import org.apache.tez.dag.api.DAG; import org.apache.tez.dag.api.Edge; import org.apache.tez.dag.api.EdgeProperty; @@ -671,7 +671,7 @@ public class TestMRRJobsDAGApi { .getConfigurationBytes()); try { - RuntimeUtils.getClazz(RELOCALIZATION_TEST_CLASS_NAME); + ReflectionUtils.getClazz(RELOCALIZATION_TEST_CLASS_NAME); LOG.info("Class found"); FileSystem fs = FileSystem.get(conf); fs.mkdirs(new Path("/tmp/relocalizationfilefound"));