tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjf...@apache.org
Subject [1/2] tez git commit: TEZ-2745. ClassNotFoundException of user code should fail dag (zjffdu)
Date Fri, 04 Sep 2015 13:19:04 GMT
Repository: tez
Updated Branches:
  refs/heads/master 7d412b203 -> 1b30b17db


http://git-wip-us.apache.org/repos/asf/tez/blob/1b30b17d/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
index c79da5d..3382a9b 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
@@ -44,7 +44,6 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
 
-import com.google.common.base.Throwables;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.tez.runtime.api.TaskContext;
 import org.apache.tez.runtime.api.impl.TezProcessorContextImpl;
@@ -59,6 +58,7 @@ import org.apache.tez.dag.api.InputDescriptor;
 import org.apache.tez.dag.api.OutputDescriptor;
 import org.apache.tez.dag.api.ProcessorDescriptor;
 import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.runtime.api.AbstractLogicalIOProcessor;
@@ -487,7 +487,7 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask {
     return false;
   }
 
-  private void initializeGroupInputs() {
+  private void initializeGroupInputs() throws TezException {
     if (groupInputSpecs != null && !groupInputSpecs.isEmpty()) {
      groupInputsMap = new ConcurrentHashMap<String, MergedLogicalInput>(groupInputSpecs.size());
      for (GroupInputSpec groupInputSpec : groupInputSpecs) {
@@ -560,7 +560,7 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask {
     return processorContext;
   }
 
-  private LogicalInput createInput(InputSpec inputSpec, InputContext inputContext) {
+  private LogicalInput createInput(InputSpec inputSpec, InputContext inputContext) throws
TezException {
     LOG.info("Creating Input");
     InputDescriptor inputDesc = inputSpec.getInputDescriptor();
     Input input = ReflectionUtils.createClazzInstance(inputDesc.getClassName(),
@@ -576,14 +576,14 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask {
 
   private LogicalInput createMergedInput(InputDescriptor inputDesc,
                                          MergedInputContext mergedInputContext,
-                                         List<Input> constituentInputs) {
+                                         List<Input> constituentInputs) throws TezException
{
     LogicalInput input = ReflectionUtils.createClazzInstance(inputDesc.getClassName(),
         new Class[]{MergedInputContext.class, List.class},
         new Object[]{mergedInputContext, constituentInputs});
     return input;
   }
 
-  private LogicalOutput createOutput(OutputSpec outputSpec, OutputContext outputContext)
{
+  private LogicalOutput createOutput(OutputSpec outputSpec, OutputContext outputContext)
throws TezException {
     LOG.info("Creating Output");
     OutputDescriptor outputDesc = outputSpec.getOutputDescriptor();
     Output output = ReflectionUtils.createClazzInstance(outputDesc.getClassName(),
@@ -599,7 +599,7 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask {
   }
 
   private AbstractLogicalIOProcessor createProcessor(
-      String processorClassName, ProcessorContext processorContext) {
+      String processorClassName, ProcessorContext processorContext) throws TezException {
     Processor processor = ReflectionUtils.createClazzInstance(processorClassName,
         new Class[]{ProcessorContext.class}, new Object[]{processorContext});
     if (!(processor instanceof AbstractLogicalIOProcessor)) {

http://git-wip-us.apache.org/repos/asf/tez/blob/1b30b17d/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
index bb6184e..2622b1f 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.tez.common.ReflectionUtils;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.EntityDescriptor;
+import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.runtime.api.MemoryUpdateCallback;
 import org.apache.tez.runtime.api.InputContext;
@@ -100,8 +101,9 @@ public class MemoryDistributor {
   /**
    * Used by the Tez framework to distribute initial memory after components
    * have made their initial requests.
+   * @throws TezException
    */
-  public void makeInitialAllocations() {
+  public void makeInitialAllocations() throws TezException {
     Preconditions.checkState(numInputsSeen.get() == numTotalInputs, "All inputs are expected
to ask for memory");
     Preconditions.checkState(numOutputsSeen.get() == numTotalOutputs, "All outputs are expected
to ask for memory");
     Iterable<InitialMemoryRequestContext> requestContexts = Iterables.transform(requestList,

http://git-wip-us.apache.org/repos/asf/tez/blob/1b30b17d/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java
b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java
index 951a877..7fbd87f 100644
--- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java
+++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java
@@ -28,6 +28,7 @@ import org.apache.tez.dag.api.InputDescriptor;
 import org.apache.tez.dag.api.OutputDescriptor;
 import org.apache.tez.dag.api.ProcessorDescriptor;
 import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
 import org.apache.tez.runtime.api.MemoryUpdateCallback;
 import org.apache.tez.runtime.api.InputContext;
 import org.apache.tez.runtime.api.OutputContext;
@@ -47,7 +48,7 @@ public class TestMemoryDistributor {
   }
   
   @Test(timeout = 5000)
-  public void testScalingNoProcessor() {
+  public void testScalingNoProcessor() throws TezException {
     MemoryDistributor dist = new MemoryDistributor(2, 1, conf);
     
     dist.setJvmMemory(10000l);
@@ -81,7 +82,7 @@ public class TestMemoryDistributor {
   }
   
   @Test(timeout = 5000)
-  public void testScalingNoProcessor2() {
+  public void testScalingNoProcessor2() throws TezException {
     // Real world values
     MemoryDistributor dist = new MemoryDistributor(2, 0, conf);
     
@@ -106,7 +107,7 @@ public class TestMemoryDistributor {
   }
   
   @Test(timeout = 5000)
-  public void testScalingProcessor() {
+  public void testScalingProcessor() throws TezException {
     MemoryDistributor dist = new MemoryDistributor(2, 1, conf);
     
     dist.setJvmMemory(10000l);
@@ -148,7 +149,7 @@ public class TestMemoryDistributor {
   }
   
   @Test(timeout = 5000)
-  public void testScalingDisabled() {
+  public void testScalingDisabled() throws TezException {
     // Real world values
     Configuration conf = new Configuration(this.conf);
     conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ENABLED, false);
@@ -175,7 +176,7 @@ public class TestMemoryDistributor {
   }
   
   @Test(timeout = 5000)
-  public void testReserveFractionConfigured() {
+  public void testReserveFractionConfigured() throws TezException {
     Configuration conf = new Configuration(this.conf);
     conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.5d);
     MemoryDistributor dist = new MemoryDistributor(2, 1, conf);

http://git-wip-us.apache.org/repos/asf/tez/blob/1b30b17d/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
index ce9095a..2123757 100644
--- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
+++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
@@ -43,6 +43,7 @@ import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -51,7 +52,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.tez.dag.api.ProcessorDescriptor;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.TezReflectionException;
 import org.apache.tez.dag.api.UserPayload;
 import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.dag.records.TezTaskAttemptID;
@@ -237,11 +238,11 @@ public class TestTaskExecution2 {
 
       TaskRunner2Result result = taskRunnerFuture.get();
       verifyTaskRunnerResult(result, EndReason.TASK_ERROR,
-          new TezUncheckedException("Unchecked exception"), false);
+          new TezReflectionException("TezReflectionException"), false);
 
       assertNull(taskReporter.currentCallable);
       umbilical.verifyTaskFailedEvent("Failure while running task",
-          ":org.apache.tez.dag.api.TezUncheckedException: "
+          ":org.apache.tez.dag.api.TezReflectionException: "
               + "Unable to load class: NotExitedProcessor");
       // Failure detected as a result of fall off from the run method. abort isn't required.
       assertFalse(TestProcessor.wasAborted());

http://git-wip-us.apache.org/repos/asf/tez/blob/1b30b17d/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java
index b34accd..a38497c 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.tez.dag.api.InputDescriptor;
 import org.apache.tez.dag.api.OutputDescriptor;
 import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
 import org.apache.tez.runtime.api.LogicalInput;
 import org.apache.tez.runtime.api.LogicalOutput;
 import org.apache.tez.runtime.api.MemoryUpdateCallback;
@@ -51,7 +52,7 @@ public class TestWeightedScalingMemoryDistributor extends TestMemoryDistributor
   }
   
   @Test(timeout = 5000)
-  public void testSimpleWeightedScaling() {
+  public void testSimpleWeightedScaling() throws TezException {
     Configuration conf = new Configuration(this.conf);
     conf.setStrings(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS,
         WeightedScalingMemoryDistributor.generateWeightStrings(0, 0, 1, 2, 3, 1, 1));
@@ -98,7 +99,7 @@ public class TestWeightedScalingMemoryDistributor extends TestMemoryDistributor
   }
 
   @Test(timeout = 5000)
-  public void testAdditionalReserveFractionWeightedScaling() {
+  public void testAdditionalReserveFractionWeightedScaling() throws TezException {
     Configuration conf = new Configuration(this.conf);
     conf.setStrings(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS,
         WeightedScalingMemoryDistributor.generateWeightStrings(0, 0, 2, 3, 6, 1, 1));

http://git-wip-us.apache.org/repos/asf/tez/blob/1b30b17d/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
index 5663e62..db1b1e1 100644
--- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
+++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
@@ -90,6 +90,7 @@ import org.apache.tez.dag.api.OutputDescriptor;
 import org.apache.tez.dag.api.ProcessorDescriptor;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezReflectionException;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.UserPayload;
 import org.apache.tez.dag.api.Vertex;
@@ -887,7 +888,7 @@ public class TestMRRJobsDAGApi {
         LOG.info("Class found");
         FileSystem fs = FileSystem.get(conf);
         fs.mkdirs(new Path("/tmp/relocalizationfilefound"));
-      } catch (TezUncheckedException e) {
+      } catch (TezReflectionException e) {
         LOG.info("Class not found");
       }
 


Mime
View raw message