tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject [1/2] TEZ-1119. Support display of user payloads in Tez UI. (hitesh)
Date Wed, 16 Jul 2014 22:20:19 GMT
Repository: incubator-tez
Updated Branches:
  refs/heads/master 171da568e -> 36d08fe03


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestRuntimeUtils.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestRuntimeUtils.java
b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestRuntimeUtils.java
deleted file mode 100644
index f374cc0..0000000
--- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestRuntimeUtils.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.runtime;
-
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.net.URL;
-import java.util.Collections;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.tez.common.RuntimeUtils;
-import org.apache.tez.dag.api.TezException;
-import org.junit.Test;
-
-public class TestRuntimeUtils {
-
-  @Test
-  public void testAddResourceToClasspath() throws IOException, TezException {
-
-    String rsrcName = "dummyfile.xml";
-    FileSystem localFs = FileSystem.getLocal(new Configuration());
-    Path p = new Path(rsrcName);
-    p = localFs.makeQualified(p);
-
-    localFs.delete(p, false);
-
-    try {
-      URL loadedUrl = null;
-
-      loadedUrl = Thread.currentThread().getContextClassLoader().getResource(rsrcName);
-      assertNull(loadedUrl);
-
-      // Add parent to classpath since we're not adding a jar
-      assertTrue(localFs.createNewFile(p));
-      String urlForm = p.toUri().toURL().toString();
-      urlForm = urlForm.substring(0, urlForm.lastIndexOf('/') + 1);
-      URL url = new URL(urlForm);
-
-      RuntimeUtils.addResourcesToClasspath(Collections.singletonList(url));
-
-      loadedUrl = Thread.currentThread().getContextClassLoader().getResource(rsrcName);
-
-      assertNotNull(loadedUrl);
-    } finally {
-      localFs.delete(p, false);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java
index e37bbe3..ac88865 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java
@@ -25,6 +25,7 @@ import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.tez.common.TezCommonUtils;
 import org.apache.tez.common.TezUtils;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.runtime.api.Event;
@@ -90,7 +91,7 @@ public class ShuffleInputEventHandler {
     }
     if (shufflePayload.hasEmptyPartitions()) {
       try {
-        byte[] emptyPartitions = TezUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions());
+        byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions());
         BitSet emptyPartitionsBitSet = TezUtils.fromByteArray(emptyPartitions);
         if (emptyPartitionsBitSet.get(partitionId)) {
           LOG.info("Source partition: " + partitionId + " did not generate any data. SrcAttempt:
["

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java
index 2d51a7d..4e3e94b 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.tez.common.TezCommonUtils;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezUtils;
 import org.apache.tez.common.counters.TaskCounter;
@@ -477,7 +478,7 @@ public class UnorderedPartitionedKVWriter extends BaseUnorderedPartitionedKVWrit
     }
     if (emptyPartitions.cardinality() != 0) {
       // Empty partitions exist
-      ByteString emptyPartitionsByteString = TezUtils.compressByteArrayToByteString(TezUtils
+      ByteString emptyPartitionsByteString = TezCommonUtils.compressByteArrayToByteString(TezUtils
           .toByteArray(emptyPartitions));
       payloadBuidler.setEmptyPartitions(emptyPartitionsByteString);
     }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
index 8fbac92..c1f5ddb 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.tez.common.TezCommonUtils;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezRuntimeFrameworkConfigs;
 import org.apache.tez.common.TezUtils;
@@ -165,7 +166,7 @@ public class OnFileSortedOutput extends AbstractLogicalOutput {
       }
       if (emptyPartitions > 0) {
         ByteString emptyPartitionsBytesString =
-            TezUtils.compressByteArrayToByteString(TezUtils.toByteArray(emptyPartitionDetails));
+            TezCommonUtils.compressByteArrayToByteString(TezUtils.toByteArray(emptyPartitionDetails));
         payloadBuilder.setEmptyPartitions(emptyPartitionsBytesString);
         LOG.info("EmptyPartition bitsetSize=" + emptyPartitionDetails.cardinality() + ",
numOutputs="
                 + getNumPhysicalOutputs() + ", emptyPartitions=" + emptyPartitions

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java
index 646ddbd..8cd47c1 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.tez.common.TezCommonUtils;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezRuntimeFrameworkConfigs;
 import org.apache.tez.common.TezUtils;
@@ -134,7 +135,7 @@ public class OnFileUnorderedKVOutput extends AbstractLogicalOutput {
       BitSet emptyPartitions = new BitSet();
       emptyPartitions.set(0);
       ByteString emptyPartitionsBytesString =
-          TezUtils.compressByteArrayToByteString(TezUtils.toByteArray(emptyPartitions));
+          TezCommonUtils.compressByteArrayToByteString(TezUtils.toByteArray(emptyPartitions));
       payloadBuilder.setEmptyPartitions(emptyPartitionsBytesString);
     }
     if (outputGenerated) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java
index 628298a..65dba32 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java
@@ -26,6 +26,7 @@ import java.util.List;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.tez.common.TezCommonUtils;
 import org.apache.tez.common.TezUtils;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.runtime.api.Event;
@@ -101,7 +102,7 @@ public class ShuffleInputEventHandlerImpl implements ShuffleEventHandler
{
         + stringify(shufflePayload));
 
     if (shufflePayload.hasEmptyPartitions()) {
-      byte[] emptyPartitions = TezUtils.decompressByteStringToByteArray(shufflePayload
+      byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload
           .getEmptyPartitions());
       BitSet emptyPartionsBitSet = TezUtils.fromByteArray(emptyPartitions);
       if (emptyPartionsBitSet.get(srcIndex)) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
index f45a7d4..5c3a83a 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
@@ -26,7 +26,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.tez.common.RuntimeUtils;
+import org.apache.tez.common.ReflectionUtils;
 import org.apache.tez.common.TezUtils;
 import org.apache.tez.dag.api.EdgeManager;
 import org.apache.tez.dag.api.EdgeManagerContext;
@@ -125,7 +125,7 @@ public class TestShuffleVertexManager {
           newEdgeManagers.clear();
           for (Entry<String, EdgeManagerDescriptor> entry :
               ((Map<String, EdgeManagerDescriptor>)invocation.getArguments()[2]).entrySet())
{
-            EdgeManager edgeManager = RuntimeUtils.createClazzInstance(
+            EdgeManager edgeManager = ReflectionUtils.createClazzInstance(
                 entry.getValue().getClassName());
             final byte[] userPayload = entry.getValue().getUserPayload();
             edgeManager.initialize(new EdgeManagerContext() {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java
index f5fd276..9b37c78 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.DefaultCodec;
 import org.apache.hadoop.util.DiskChecker.DiskErrorException;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.common.TezCommonUtils;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezRuntimeFrameworkConfigs;
 import org.apache.tez.common.TezUtils;
@@ -316,7 +317,7 @@ public class TestUnorderedPartitionedKVWriter {
     BitSet emptyPartitionBits = null;
     if (partitionsWithData.cardinality() != numPartitions) {
       assertTrue(eventProto.hasEmptyPartitions());
-      byte[] emptyPartitions = TezUtils.decompressByteStringToByteArray(eventProto
+      byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(eventProto
           .getEmptyPartitions());
       emptyPartitionBits = TezUtils.fromByteArray(emptyPartitions);
       assertEquals(numPartitions - partitionsWithData.cardinality(),
@@ -505,7 +506,7 @@ public class TestUnorderedPartitionedKVWriter {
       emptyPartitionBits = new BitSet(numPartitions);
     } else {
       assertTrue(eventProto.hasEmptyPartitions());
-      byte[] emptyPartitions = TezUtils.decompressByteStringToByteArray(eventProto
+      byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(eventProto
           .getEmptyPartitions());
       emptyPartitionBits = TezUtils.fromByteArray(emptyPartitions);
       if (numRecordsWritten == 0) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java
index e95c30d..2aa7396 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java
@@ -27,6 +27,7 @@ import java.util.BitSet;
 import java.util.LinkedList;
 import java.util.List;
 
+import org.apache.tez.common.TezCommonUtils;
 import org.apache.tez.common.TezUtils;
 import org.apache.tez.runtime.api.Event;
 import org.apache.tez.runtime.api.TezInputContext;
@@ -151,7 +152,7 @@ public class TestShuffleInputEventHandlerImpl {
     for (int i : emptyPartitions) {
       bitSet.set(i);
     }
-    ByteString emptyPartitionsBytesString = TezUtils.compressByteArrayToByteString(TezUtils
+    ByteString emptyPartitionsBytesString = TezCommonUtils.compressByteArrayToByteString(TezUtils
         .toByteArray(bitSet));
     return emptyPartitionsBytesString;
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36d08fe0/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
index e5667c7..ea0378d 100644
--- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
+++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
@@ -67,7 +67,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.client.TezClientUtils;
 import org.apache.tez.client.TezClient;
 import org.apache.tez.client.TezAppMasterStatus;
-import org.apache.tez.common.RuntimeUtils;
+import org.apache.tez.common.ReflectionUtils;
 import org.apache.tez.dag.api.DAG;
 import org.apache.tez.dag.api.Edge;
 import org.apache.tez.dag.api.EdgeProperty;
@@ -671,7 +671,7 @@ public class TestMRRJobsDAGApi {
           .getConfigurationBytes());
 
       try {
-        RuntimeUtils.getClazz(RELOCALIZATION_TEST_CLASS_NAME);
+        ReflectionUtils.getClazz(RELOCALIZATION_TEST_CLASS_NAME);
         LOG.info("Class found");
         FileSystem fs = FileSystem.get(conf);
         fs.mkdirs(new Path("/tmp/relocalizationfilefound"));


Mime
View raw message