Return-Path: X-Original-To: apmail-tez-commits-archive@minotaur.apache.org Delivered-To: apmail-tez-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2617E1013B for ; Mon, 23 Sep 2013 17:47:35 +0000 (UTC) Received: (qmail 89264 invoked by uid 500); 23 Sep 2013 17:47:28 -0000 Delivered-To: apmail-tez-commits-archive@tez.apache.org Received: (qmail 89241 invoked by uid 500); 23 Sep 2013 17:47:27 -0000 Mailing-List: contact commits-help@tez.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.incubator.apache.org Delivered-To: mailing list commits@tez.incubator.apache.org Received: (qmail 89229 invoked by uid 99); 23 Sep 2013 17:47:26 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 23 Sep 2013 17:47:26 +0000 X-ASF-Spam-Status: No, hits=-2002.3 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Mon, 23 Sep 2013 17:46:26 +0000 Received: (qmail 83250 invoked by uid 99); 23 Sep 2013 17:45:17 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 23 Sep 2013 17:45:17 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 83A039076DF; Mon, 23 Sep 2013 17:45:15 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sseth@apache.org To: commits@tez.incubator.apache.org Date: Mon, 23 Sep 2013 17:45:25 -0000 Message-Id: <7d5a384be2444557b39d9d80ab807269@git.apache.org> In-Reply-To: <443f608461004be4a7082ca4f304c566@git.apache.org> References: <443f608461004be4a7082ca4f304c566@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [11/20] TEZ-443. Merge tez-dag-api and tez-engine-api into a single module - tez-api (part of TEZ-398). (sseth) X-Virus-Checked: Checked by ClamAV on apache.org http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/common/TezEngineUtils.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/TezEngineUtils.java b/tez-engine/src/main/java/org/apache/tez/engine/common/TezEngineUtils.java index ab78d82..cc29e94 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/common/TezEngineUtils.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/common/TezEngineUtils.java @@ -29,11 +29,11 @@ import org.apache.tez.common.Constants; import org.apache.tez.common.TezJobConfig; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.engine.api.Partitioner; +import org.apache.tez.engine.api.TezOutputContext; +import org.apache.tez.engine.api.TezTaskContext; import org.apache.tez.engine.common.combine.Combiner; import org.apache.tez.engine.common.task.local.output.TezTaskOutput; import org.apache.tez.engine.common.task.local.output.TezTaskOutputFiles; -import org.apache.tez.engine.newapi.TezOutputContext; -import org.apache.tez.engine.newapi.TezTaskContext; public class TezEngineUtils { http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/common/combine/Combiner.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/combine/Combiner.java b/tez-engine/src/main/java/org/apache/tez/engine/common/combine/Combiner.java index 6f86d61..b387b36 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/common/combine/Combiner.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/common/combine/Combiner.java @@ -23,7 +23,6 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.tez.common.TezJobConfig; -import org.apache.tez.common.TezTaskContext; import org.apache.tez.engine.common.sort.impl.IFile.Writer; import org.apache.tez.engine.common.sort.impl.TezRawKeyValueIterator; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/common/localshuffle/LocalShuffle.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/localshuffle/LocalShuffle.java b/tez-engine/src/main/java/org/apache/tez/engine/common/localshuffle/LocalShuffle.java index 1cb89a7..546151f 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/common/localshuffle/LocalShuffle.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/common/localshuffle/LocalShuffle.java @@ -32,12 +32,12 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.counters.TaskCounter; import org.apache.tez.common.counters.TezCounter; +import org.apache.tez.engine.api.TezInputContext; import org.apache.tez.engine.common.ConfigUtils; import org.apache.tez.engine.common.sort.impl.TezMerger; import org.apache.tez.engine.common.sort.impl.TezRawKeyValueIterator; import org.apache.tez.engine.common.task.local.output.TezLocalTaskOutputFiles; import org.apache.tez.engine.common.task.local.output.TezTaskOutput; -import org.apache.tez.engine.newapi.TezInputContext; @SuppressWarnings({"rawtypes"}) public class LocalShuffle { http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java index 5c71644..b48bb0b 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java @@ -46,12 +46,12 @@ import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.util.ReflectionUtils; import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.counters.TezCounter; +import org.apache.tez.engine.api.TezInputContext; import org.apache.tez.engine.common.ConfigUtils; import org.apache.tez.engine.common.InputAttemptIdentifier; import org.apache.tez.engine.common.security.SecureShuffleUtils; import org.apache.tez.engine.common.shuffle.impl.MapOutput.Type; import org.apache.tez.engine.common.sort.impl.IFileInputStream; -import org.apache.tez.engine.newapi.TezInputContext; import com.google.common.annotations.VisibleForTesting; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java index bf2be4e..b8792fb 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java @@ -46,6 +46,7 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.tez.common.Constants; import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.counters.TezCounter; +import org.apache.tez.engine.api.TezInputContext; import org.apache.tez.engine.common.ConfigUtils; import org.apache.tez.engine.common.InputAttemptIdentifier; import org.apache.tez.engine.common.combine.Combiner; @@ -56,7 +57,6 @@ import org.apache.tez.engine.common.sort.impl.IFile.Writer; import org.apache.tez.engine.common.sort.impl.TezMerger.Segment; import org.apache.tez.engine.common.task.local.output.TezTaskOutputFiles; import org.apache.tez.engine.hadoop.compat.NullProgressable; -import org.apache.tez.engine.newapi.TezInputContext; @InterfaceAudience.Private @InterfaceStability.Unstable http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java index f605b7c..15332a1 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java @@ -38,12 +38,12 @@ import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.counters.TaskCounter; import org.apache.tez.common.counters.TezCounter; import org.apache.tez.dag.api.TezUncheckedException; +import org.apache.tez.engine.api.Event; +import org.apache.tez.engine.api.TezInputContext; import org.apache.tez.engine.common.TezEngineUtils; import org.apache.tez.engine.common.combine.Combiner; import org.apache.tez.engine.common.shuffle.server.ShuffleHandler; import org.apache.tez.engine.common.sort.impl.TezRawKeyValueIterator; -import org.apache.tez.engine.newapi.Event; -import org.apache.tez.engine.newapi.TezInputContext; import org.apache.tez.engine.shuffle.common.ShuffleUtils; import com.google.common.base.Preconditions; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleInputEventHandler.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleInputEventHandler.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleInputEventHandler.java index 98c7ab1..a8e5fe4 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleInputEventHandler.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleInputEventHandler.java @@ -24,14 +24,14 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tez.dag.api.TezUncheckedException; +import org.apache.tez.engine.api.Event; +import org.apache.tez.engine.api.TezInputContext; +import org.apache.tez.engine.api.events.DataMovementEvent; +import org.apache.tez.engine.api.events.InputFailedEvent; +import org.apache.tez.engine.api.events.InputInformationEvent; import org.apache.tez.engine.common.InputAttemptIdentifier; import org.apache.tez.engine.common.shuffle.newimpl.ShuffleUserPayloads.DataMovementEventPayloadProto; import org.apache.tez.engine.common.shuffle.newimpl.ShuffleUserPayloads.InputInformationEventPayloadProto; -import org.apache.tez.engine.newapi.Event; -import org.apache.tez.engine.newapi.TezInputContext; -import org.apache.tez.engine.newapi.events.DataMovementEvent; -import org.apache.tez.engine.newapi.events.InputFailedEvent; -import org.apache.tez.engine.newapi.events.InputInformationEvent; import com.google.common.base.Preconditions; import com.google.protobuf.InvalidProtocolBufferException; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java index acbd054..be75668 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java @@ -40,11 +40,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IntWritable; import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.counters.TezCounter; +import org.apache.tez.engine.api.Event; +import org.apache.tez.engine.api.TezInputContext; +import org.apache.tez.engine.api.events.InputReadErrorEvent; import org.apache.tez.engine.common.InputAttemptIdentifier; import org.apache.tez.engine.common.TezEngineUtils; -import org.apache.tez.engine.newapi.Event; -import org.apache.tez.engine.newapi.TezInputContext; -import org.apache.tez.engine.newapi.events.InputReadErrorEvent; import com.google.common.collect.Lists; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/server/ShuffleHandler.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/server/ShuffleHandler.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/server/ShuffleHandler.java index f61670e..a6d1c5b 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/server/ShuffleHandler.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/server/ShuffleHandler.java @@ -64,12 +64,12 @@ import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext; import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext; import org.apache.hadoop.yarn.server.api.AuxiliaryService; import org.apache.tez.common.TezJobConfig; +import org.apache.tez.engine.api.TezOutputContext; import org.apache.tez.engine.common.security.JobTokenIdentifier; import org.apache.tez.engine.common.security.JobTokenSecretManager; import org.apache.tez.engine.common.security.SecureShuffleUtils; import org.apache.tez.engine.common.shuffle.impl.ShuffleHeader; import org.apache.tez.engine.common.sort.impl.ExternalSorter; -import org.apache.tez.engine.newapi.TezOutputContext; import org.apache.tez.engine.shuffle.common.ShuffleUtils; import org.jboss.netty.bootstrap.ServerBootstrap; import org.jboss.netty.buffer.ChannelBuffers; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java index 8b4bd4e..5aa0ddf 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java @@ -43,6 +43,7 @@ import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.counters.TaskCounter; import org.apache.tez.common.counters.TezCounter; import org.apache.tez.engine.api.Partitioner; +import org.apache.tez.engine.api.TezOutputContext; import org.apache.tez.engine.common.ConfigUtils; import org.apache.tez.engine.common.TezEngineUtils; import org.apache.tez.engine.common.combine.Combiner; @@ -50,7 +51,6 @@ import org.apache.tez.engine.common.shuffle.impl.ShuffleHeader; import org.apache.tez.engine.common.sort.impl.IFile.Writer; import org.apache.tez.engine.common.task.local.output.TezTaskOutput; import org.apache.tez.engine.hadoop.compat.NullProgressable; -import org.apache.tez.engine.newapi.TezOutputContext; @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class ExternalSorter { http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java index 952568e..1bf17a3 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java @@ -44,10 +44,10 @@ import org.apache.hadoop.util.IndexedSortable; import org.apache.hadoop.util.IndexedSorter; import org.apache.hadoop.util.Progress; import org.apache.tez.common.TezJobConfig; +import org.apache.tez.engine.api.TezOutputContext; import org.apache.tez.engine.common.ConfigUtils; import org.apache.tez.engine.common.sort.impl.IFile.Writer; import org.apache.tez.engine.common.sort.impl.TezMerger.Segment; -import org.apache.tez.engine.newapi.TezOutputContext; @SuppressWarnings({"unchecked", "rawtypes"}) public class PipelinedSorter extends ExternalSorter { http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java index 1ad31f7..6b48270 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java @@ -40,6 +40,7 @@ import org.apache.hadoop.util.IndexedSortable; import org.apache.hadoop.util.Progress; import org.apache.hadoop.util.StringUtils; import org.apache.tez.common.TezJobConfig; +import org.apache.tez.engine.api.TezOutputContext; import org.apache.tez.engine.common.ConfigUtils; import org.apache.tez.engine.common.sort.impl.ExternalSorter; import org.apache.tez.engine.common.sort.impl.IFile; @@ -49,7 +50,6 @@ import org.apache.tez.engine.common.sort.impl.TezRawKeyValueIterator; import org.apache.tez.engine.common.sort.impl.TezSpillRecord; import org.apache.tez.engine.common.sort.impl.IFile.Writer; import org.apache.tez.engine.common.sort.impl.TezMerger.Segment; -import org.apache.tez.engine.newapi.TezOutputContext; @SuppressWarnings({"unchecked", "rawtypes"}) public class DefaultSorter extends ExternalSorter implements IndexedSortable { http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/InMemoryShuffleSorter.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/InMemoryShuffleSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/InMemoryShuffleSorter.java index 53e6003..e2b3315 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/InMemoryShuffleSorter.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/InMemoryShuffleSorter.java @@ -29,10 +29,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.util.DataChecksum; +import org.apache.tez.engine.api.TezOutputContext; import org.apache.tez.engine.common.shuffle.impl.ShuffleHeader; import org.apache.tez.engine.common.shuffle.server.ShuffleHandler; import org.apache.tez.engine.common.sort.impl.IFile; -import org.apache.tez.engine.newapi.TezOutputContext; public class InMemoryShuffleSorter extends DefaultSorter { http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/lib/input/LocalMergedInput.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/lib/input/LocalMergedInput.java b/tez-engine/src/main/java/org/apache/tez/engine/lib/input/LocalMergedInput.java index 8ae6bfe..ed57c61 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/lib/input/LocalMergedInput.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/lib/input/LocalMergedInput.java @@ -22,10 +22,10 @@ import java.util.Collections; import java.util.List; import org.apache.tez.common.TezUtils; +import org.apache.tez.engine.api.Event; +import org.apache.tez.engine.api.LogicalInput; +import org.apache.tez.engine.api.TezInputContext; import org.apache.tez.engine.common.localshuffle.LocalShuffle; -import org.apache.tez.engine.newapi.Event; -import org.apache.tez.engine.newapi.LogicalInput; -import org.apache.tez.engine.newapi.TezInputContext; /** * LocalMergedInput in an {@link LogicalInput} which shuffles intermediate http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/lib/input/ShuffledMergedInput.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/lib/input/ShuffledMergedInput.java b/tez-engine/src/main/java/org/apache/tez/engine/lib/input/ShuffledMergedInput.java index c719fba..3db0632 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/lib/input/ShuffledMergedInput.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/lib/input/ShuffledMergedInput.java @@ -29,14 +29,14 @@ import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.TezUtils; import org.apache.tez.common.counters.TaskCounter; import org.apache.tez.common.counters.TezCounter; +import org.apache.tez.engine.api.Event; import org.apache.tez.engine.api.KVReader; +import org.apache.tez.engine.api.LogicalInput; +import org.apache.tez.engine.api.TezInputContext; import org.apache.tez.engine.common.ConfigUtils; import org.apache.tez.engine.common.ValuesIterator; import org.apache.tez.engine.common.shuffle.impl.Shuffle; import org.apache.tez.engine.common.sort.impl.TezRawKeyValueIterator; -import org.apache.tez.engine.newapi.Event; -import org.apache.tez.engine.newapi.LogicalInput; -import org.apache.tez.engine.newapi.TezInputContext; /** * ShuffleMergedInput in a {@link LogicalInput} which shuffles http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/lib/input/ShuffledUnorderedKVInput.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/lib/input/ShuffledUnorderedKVInput.java b/tez-engine/src/main/java/org/apache/tez/engine/lib/input/ShuffledUnorderedKVInput.java index c2126ca..44238fd 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/lib/input/ShuffledUnorderedKVInput.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/lib/input/ShuffledUnorderedKVInput.java @@ -23,11 +23,11 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.TezUtils; +import org.apache.tez.engine.api.Event; +import org.apache.tez.engine.api.LogicalInput; +import org.apache.tez.engine.api.Reader; +import org.apache.tez.engine.api.TezInputContext; import org.apache.tez.engine.broadcast.input.BroadcastShuffleManager; -import org.apache.tez.engine.newapi.Event; -import org.apache.tez.engine.newapi.LogicalInput; -import org.apache.tez.engine.newapi.Reader; -import org.apache.tez.engine.newapi.TezInputContext; import com.google.common.base.Preconditions; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/lib/output/InMemorySortedOutput.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/lib/output/InMemorySortedOutput.java b/tez-engine/src/main/java/org/apache/tez/engine/lib/output/InMemorySortedOutput.java index 218aa21..26a01c8 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/lib/output/InMemorySortedOutput.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/lib/output/InMemorySortedOutput.java @@ -22,13 +22,13 @@ import java.util.Collections; import java.util.List; import org.apache.tez.common.TezUtils; +import org.apache.tez.engine.api.Event; import org.apache.tez.engine.api.KVWriter; +import org.apache.tez.engine.api.LogicalOutput; +import org.apache.tez.engine.api.Output; +import org.apache.tez.engine.api.TezOutputContext; +import org.apache.tez.engine.api.Writer; import org.apache.tez.engine.common.sort.impl.dflt.InMemoryShuffleSorter; -import org.apache.tez.engine.newapi.Event; -import org.apache.tez.engine.newapi.LogicalOutput; -import org.apache.tez.engine.newapi.Output; -import org.apache.tez.engine.newapi.TezOutputContext; -import org.apache.tez.engine.newapi.Writer; /** * {@link InMemorySortedOutput} is an {@link Output} which sorts key/value pairs http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java b/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java index 963276d..b24e10d 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java @@ -25,8 +25,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tez.engine.api.Event; import org.apache.tez.engine.common.task.local.output.TezTaskOutput; -import org.apache.tez.engine.newapi.Event; public class LocalOnFileSorterOutput extends OnFileSortedOutput { http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/lib/output/OnFileSortedOutput.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/lib/output/OnFileSortedOutput.java b/tez-engine/src/main/java/org/apache/tez/engine/lib/output/OnFileSortedOutput.java index 7e0ca37..685722e 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/lib/output/OnFileSortedOutput.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/lib/output/OnFileSortedOutput.java @@ -26,14 +26,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.TezUtils; +import org.apache.tez.engine.api.Event; import org.apache.tez.engine.api.KVWriter; +import org.apache.tez.engine.api.LogicalOutput; +import org.apache.tez.engine.api.TezOutputContext; +import org.apache.tez.engine.api.events.DataMovementEvent; import org.apache.tez.engine.common.shuffle.newimpl.ShuffleUserPayloads.DataMovementEventPayloadProto; import org.apache.tez.engine.common.sort.impl.ExternalSorter; import org.apache.tez.engine.common.sort.impl.dflt.DefaultSorter; -import org.apache.tez.engine.newapi.Event; -import org.apache.tez.engine.newapi.LogicalOutput; -import org.apache.tez.engine.newapi.TezOutputContext; -import org.apache.tez.engine.newapi.events.DataMovementEvent; import org.apache.tez.engine.shuffle.common.ShuffleUtils; import com.google.common.collect.Lists; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/lib/output/OnFileUnorderedKVOutput.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/lib/output/OnFileUnorderedKVOutput.java b/tez-engine/src/main/java/org/apache/tez/engine/lib/output/OnFileUnorderedKVOutput.java index 37edde8..3ff603f 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/lib/output/OnFileUnorderedKVOutput.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/lib/output/OnFileUnorderedKVOutput.java @@ -24,13 +24,13 @@ import java.util.List; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.tez.dag.api.TezUncheckedException; +import org.apache.tez.engine.api.Event; import org.apache.tez.engine.api.KVWriter; +import org.apache.tez.engine.api.LogicalOutput; +import org.apache.tez.engine.api.TezOutputContext; +import org.apache.tez.engine.api.events.DataMovementEvent; import org.apache.tez.engine.broadcast.output.FileBasedKVWriter; import org.apache.tez.engine.common.shuffle.newimpl.ShuffleUserPayloads.DataMovementEventPayloadProto; -import org.apache.tez.engine.newapi.Event; -import org.apache.tez.engine.newapi.LogicalOutput; -import org.apache.tez.engine.newapi.TezOutputContext; -import org.apache.tez.engine.newapi.events.DataMovementEvent; import org.apache.tez.engine.shuffle.common.ShuffleUtils; import com.google.common.base.Preconditions; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-engine/src/main/java/org/apache/tez/engine/newruntime/LogicalIOProcessorRuntimeTask.java ---------------------------------------------------------------------- diff --git a/tez-engine/src/main/java/org/apache/tez/engine/newruntime/LogicalIOProcessorRuntimeTask.java b/tez-engine/src/main/java/org/apache/tez/engine/newruntime/LogicalIOProcessorRuntimeTask.java index 77299de..bfd898b 100644 --- a/tez-engine/src/main/java/org/apache/tez/engine/newruntime/LogicalIOProcessorRuntimeTask.java +++ b/tez-engine/src/main/java/org/apache/tez/engine/newruntime/LogicalIOProcessorRuntimeTask.java @@ -38,6 +38,16 @@ import org.apache.hadoop.util.StringUtils; import org.apache.tez.dag.api.ProcessorDescriptor; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.dag.records.TezTaskAttemptID; +import org.apache.tez.engine.api.Event; +import org.apache.tez.engine.api.Input; +import org.apache.tez.engine.api.LogicalIOProcessor; +import org.apache.tez.engine.api.LogicalInput; +import org.apache.tez.engine.api.LogicalOutput; +import org.apache.tez.engine.api.Output; +import org.apache.tez.engine.api.Processor; +import org.apache.tez.engine.api.TezInputContext; +import org.apache.tez.engine.api.TezOutputContext; +import org.apache.tez.engine.api.TezProcessorContext; import org.apache.tez.engine.api.impl.EventMetaData; import org.apache.tez.engine.api.impl.InputSpec; import org.apache.tez.engine.api.impl.OutputSpec; @@ -49,16 +59,6 @@ import org.apache.tez.engine.api.impl.TezProcessorContextImpl; import org.apache.tez.engine.api.impl.TezUmbilical; import org.apache.tez.engine.api.impl.EventMetaData.EventProducerConsumerType; import org.apache.tez.engine.common.security.JobTokenIdentifier; -import org.apache.tez.engine.newapi.Event; -import org.apache.tez.engine.newapi.Input; -import org.apache.tez.engine.newapi.LogicalIOProcessor; -import org.apache.tez.engine.newapi.LogicalInput; -import org.apache.tez.engine.newapi.LogicalOutput; -import org.apache.tez.engine.newapi.Output; -import org.apache.tez.engine.newapi.Processor; -import org.apache.tez.engine.newapi.TezInputContext; -import org.apache.tez.engine.newapi.TezOutputContext; -import org.apache.tez.engine.newapi.TezProcessorContext; import org.apache.tez.engine.shuffle.common.ShuffleUtils; import com.google.common.base.Preconditions; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/pom.xml ---------------------------------------------------------------------- diff --git a/tez-mapreduce/pom.xml b/tez-mapreduce/pom.xml index ea5016b..2c6b78e 100644 --- a/tez-mapreduce/pom.xml +++ b/tez-mapreduce/pom.xml @@ -27,7 +27,7 @@ org.apache.tez - tez-dag-api + tez-api org.apache.tez http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/src/main/java/org/apache/tez/common/TezTaskStatus.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/common/TezTaskStatus.java b/tez-mapreduce/src/main/java/org/apache/tez/common/TezTaskStatus.java new file mode 100644 index 0000000..45ea80e --- /dev/null +++ b/tez-mapreduce/src/main/java/org/apache/tez/common/TezTaskStatus.java @@ -0,0 +1,105 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.tez.common; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.io.Writable; +import org.apache.tez.common.counters.TezCounters; +import org.apache.tez.dag.records.TezTaskAttemptID; + +// TODO NEWTEZ Get rid of this. +public interface TezTaskStatus extends Writable { + + //enumeration for reporting current phase of a task. + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static enum Phase{STARTING, MAP, SHUFFLE, SORT, REDUCE, CLEANUP} + + // what state is the task in? + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static enum State {RUNNING, SUCCEEDED, FAILED, UNASSIGNED, KILLED, + COMMIT_PENDING, FAILED_UNCLEAN, KILLED_UNCLEAN} + + public abstract TezTaskAttemptID getTaskAttemptId(); + + public abstract float getProgress(); + + public abstract void setProgress(float progress); + + public abstract State getRunState(); + + public abstract void setRunState(State runState); + + public abstract String getDiagnosticInfo(); + + public abstract void setDiagnosticInfo(String info); + + // TODOTEZDAG Remove stateString / rename + public abstract String getStateString(); + + public abstract void setStateString(String stateString); + + public abstract long getFinishTime(); + + public abstract void setFinishTime(long finishTime); + + // TODOTEZDAG Can shuffle / merge be made generic ? Otherwise just a single finish time. + public abstract long getShuffleFinishTime(); + + public abstract void setShuffleFinishTime(long shuffleFinishTime); + + public abstract long getMapFinishTime(); + + public abstract void setMapFinishTime(long mapFinishTime); + + public abstract long getSortFinishTime(); + + public abstract void setSortFinishTime(long sortFinishTime); + + public abstract long getStartTime(); + + public abstract void setStartTime(long startTime); + + // TODOTEZDAG Remove phase + public abstract Phase getPhase(); + + public abstract void setPhase(Phase phase); + + public abstract TezCounters getCounters(); + + public abstract void setCounters(TezCounters counters); + + public abstract List getFailedDependencies(); + + public abstract void addFailedDependency(TezTaskAttemptID taskAttempttId); + + public abstract void clearStatus(); + + public abstract void statusUpdate(float f, String string, TezCounters counters); + + // TODOTEZDAG maybe remove ? + public abstract long getLocalOutputSize(); + + public abstract void setOutputSize(long l); + +} http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java index 56e88c7..199bbfe 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java @@ -42,14 +42,14 @@ import org.apache.hadoop.util.Progress; import org.apache.hadoop.util.ReflectionUtils; import org.apache.tez.common.TezUtils; import org.apache.tez.common.counters.TezCounter; +import org.apache.tez.engine.api.TezInputContext; +import org.apache.tez.engine.api.TezOutputContext; +import org.apache.tez.engine.api.TezTaskContext; import org.apache.tez.engine.common.ConfigUtils; import org.apache.tez.engine.common.ValuesIterator; import org.apache.tez.engine.common.combine.Combiner; import org.apache.tez.engine.common.sort.impl.IFile.Writer; import org.apache.tez.engine.common.sort.impl.TezRawKeyValueIterator; -import org.apache.tez.engine.newapi.TezInputContext; -import org.apache.tez.engine.newapi.TezOutputContext; -import org.apache.tez.engine.newapi.TezTaskContext; import org.apache.tez.mapreduce.hadoop.MRConfig; import org.apache.tez.mapreduce.hadoop.MRJobConfig; import org.apache.tez.mapreduce.hadoop.mapred.MRCounters; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java index 768d347..889c64c 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java @@ -22,7 +22,6 @@ import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.TypeConverter; -import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEventStatus; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.tez.common.TezTaskStatus.Phase; import org.apache.tez.common.counters.CounterGroup; @@ -30,7 +29,6 @@ import org.apache.tez.common.counters.TezCounter; import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.records.TezDAGID; import org.apache.tez.dag.records.TezTaskAttemptID; -import org.apache.tez.engine.records.TezDependentTaskCompletionEvent; public class TezTypeConverters { @@ -66,13 +64,6 @@ public class TezTypeConverters { return tezTaskAttemptId; } - public static TezDependentTaskCompletionEvent.Status toTez( - TaskAttemptCompletionEventStatus status) { - return TezDependentTaskCompletionEvent.Status.valueOf(status.toString()); - } - - - public static Counters fromTez(TezCounters tezCounters) { if (tezCounters == null) { return null; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRReporter.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRReporter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRReporter.java index f5e08dc..2a926d7 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRReporter.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRReporter.java @@ -3,8 +3,8 @@ package org.apache.tez.mapreduce.hadoop.mapred; import org.apache.hadoop.mapred.Counters.Counter; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.Reporter; -import org.apache.tez.engine.newapi.TezProcessorContext; -import org.apache.tez.engine.newapi.TezTaskContext; +import org.apache.tez.engine.api.TezProcessorContext; +import org.apache.tez.engine.api.TezTaskContext; import org.apache.tez.mapreduce.common.Utils; public class MRReporter implements Reporter { http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/MapContextImpl.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/MapContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/MapContextImpl.java index b0348c9..dcdb3ff 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/MapContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/MapContextImpl.java @@ -30,7 +30,7 @@ import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptID; -import org.apache.tez.engine.newapi.TezTaskContext; +import org.apache.tez.engine.api.TezTaskContext; /** * The context that is given to the {@link Mapper}. http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskAttemptContextImpl.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskAttemptContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskAttemptContextImpl.java index d8548a4..4035c71 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskAttemptContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskAttemptContextImpl.java @@ -24,7 +24,7 @@ import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskID; import org.apache.hadoop.mapreduce.TaskType; -import org.apache.tez.engine.newapi.TezTaskContext; +import org.apache.tez.engine.api.TezTaskContext; import org.apache.tez.mapreduce.common.Utils; // NOTE: NEWTEZ: This is a copy of org.apache.tez.mapreduce.hadoop.mapred (not mapreduce). mapred likely does not need it's own copy of this class. http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskInputOutputContextImpl.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskInputOutputContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskInputOutputContextImpl.java index c2920dc..05ea89c 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskInputOutputContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskInputOutputContextImpl.java @@ -29,7 +29,7 @@ import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskInputOutputContext; -import org.apache.tez.engine.newapi.TezTaskContext; +import org.apache.tez.engine.api.TezTaskContext; /** * A context object that allows input and output from the task. It is only http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/SimpleInput.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/SimpleInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/SimpleInput.java index f03fdc7..598f801 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/SimpleInput.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/SimpleInput.java @@ -46,10 +46,10 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.tez.common.TezUtils; import org.apache.tez.common.counters.TaskCounter; import org.apache.tez.common.counters.TezCounter; +import org.apache.tez.engine.api.Event; import org.apache.tez.engine.api.KVReader; -import org.apache.tez.engine.newapi.Event; -import org.apache.tez.engine.newapi.LogicalInput; -import org.apache.tez.engine.newapi.TezInputContext; +import org.apache.tez.engine.api.LogicalInput; +import org.apache.tez.engine.api.TezInputContext; import org.apache.tez.mapreduce.common.Utils; import org.apache.tez.mapreduce.hadoop.mapred.MRReporter; import org.apache.tez.mapreduce.hadoop.mapreduce.TaskAttemptContextImpl; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/SimpleOutput.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/SimpleOutput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/SimpleOutput.java index dd28ab8..d82c9e2 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/SimpleOutput.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/SimpleOutput.java @@ -26,10 +26,10 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.tez.common.TezUtils; import org.apache.tez.common.counters.TaskCounter; import org.apache.tez.common.counters.TezCounter; +import org.apache.tez.engine.api.Event; import org.apache.tez.engine.api.KVWriter; -import org.apache.tez.engine.newapi.Event; -import org.apache.tez.engine.newapi.LogicalOutput; -import org.apache.tez.engine.newapi.TezOutputContext; +import org.apache.tez.engine.api.LogicalOutput; +import org.apache.tez.engine.api.TezOutputContext; import org.apache.tez.mapreduce.common.Utils; import org.apache.tez.mapreduce.hadoop.MRConfig; import org.apache.tez.mapreduce.hadoop.MRJobConfig; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java index af087c9..1a01466 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java @@ -71,12 +71,11 @@ import org.apache.tez.common.counters.TaskCounter; import org.apache.tez.common.counters.TezCounter; import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.records.TezDAGID; +import org.apache.tez.engine.api.LogicalOutput; +import org.apache.tez.engine.api.TezProcessorContext; import org.apache.tez.engine.common.security.JobTokenIdentifier; import org.apache.tez.engine.common.security.TokenCache; import org.apache.tez.engine.common.sort.impl.TezRawKeyValueIterator; -import org.apache.tez.engine.newapi.LogicalOutput; -import org.apache.tez.engine.newapi.TezProcessorContext; -import org.apache.tez.engine.records.OutputContext; import org.apache.tez.mapreduce.hadoop.DeprecatedKeys; import org.apache.tez.mapreduce.hadoop.IDConverter; import org.apache.tez.mapreduce.hadoop.MRConfig; @@ -422,14 +421,6 @@ public abstract class MRTask { InterruptedException { } - public void outputReady(MRTaskReporter reporter, OutputContext outputContext) - throws IOException, - InterruptedException { - LOG.info("Task: " + taskAttemptId + " reporting outputReady"); - updateCounters(); - statusUpdate(); - } - public void done(LogicalOutput output) throws IOException, InterruptedException { updateCounters(); http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTaskReporter.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTaskReporter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTaskReporter.java index d32b4c0..22312f7 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTaskReporter.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTaskReporter.java @@ -24,10 +24,10 @@ import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.Reporter; import org.apache.tez.common.counters.TezCounter; -import org.apache.tez.engine.newapi.TezInputContext; -import org.apache.tez.engine.newapi.TezOutputContext; -import org.apache.tez.engine.newapi.TezProcessorContext; -import org.apache.tez.engine.newapi.TezTaskContext; +import org.apache.tez.engine.api.TezInputContext; +import org.apache.tez.engine.api.TezOutputContext; +import org.apache.tez.engine.api.TezProcessorContext; +import org.apache.tez.engine.api.TezTaskContext; import org.apache.tez.mapreduce.hadoop.mapred.MRCounters; import org.apache.tez.mapreduce.hadoop.mapred.MRReporter; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java index e1ed168..2084146 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java @@ -35,14 +35,14 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.tez.common.counters.TaskCounter; import org.apache.tez.common.counters.TezCounter; import org.apache.tez.dag.api.TezException; +import org.apache.tez.engine.api.Event; import org.apache.tez.engine.api.KVReader; import org.apache.tez.engine.api.KVWriter; +import org.apache.tez.engine.api.LogicalIOProcessor; +import org.apache.tez.engine.api.LogicalInput; +import org.apache.tez.engine.api.LogicalOutput; +import org.apache.tez.engine.api.TezProcessorContext; import org.apache.tez.engine.lib.output.OnFileSortedOutput; -import org.apache.tez.engine.newapi.Event; -import org.apache.tez.engine.newapi.LogicalIOProcessor; -import org.apache.tez.engine.newapi.LogicalInput; -import org.apache.tez.engine.newapi.LogicalOutput; -import org.apache.tez.engine.newapi.TezProcessorContext; import org.apache.tez.mapreduce.hadoop.mapreduce.MapContextImpl; import org.apache.tez.mapreduce.input.SimpleInput; import org.apache.tez.mapreduce.input.SimpleInputLegacy; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java index b7288e8..9210187 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java @@ -37,16 +37,16 @@ import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; import org.apache.tez.common.counters.TaskCounter; import org.apache.tez.common.counters.TezCounter; +import org.apache.tez.engine.api.Event; import org.apache.tez.engine.api.KVReader; import org.apache.tez.engine.api.KVWriter; +import org.apache.tez.engine.api.LogicalIOProcessor; +import org.apache.tez.engine.api.LogicalInput; +import org.apache.tez.engine.api.LogicalOutput; +import org.apache.tez.engine.api.TezProcessorContext; import org.apache.tez.engine.common.ConfigUtils; import org.apache.tez.engine.common.sort.impl.TezRawKeyValueIterator; import org.apache.tez.engine.lib.output.OnFileSortedOutput; -import org.apache.tez.engine.newapi.Event; -import org.apache.tez.engine.newapi.LogicalIOProcessor; -import org.apache.tez.engine.newapi.LogicalInput; -import org.apache.tez.engine.newapi.LogicalOutput; -import org.apache.tez.engine.newapi.TezProcessorContext; import org.apache.tez.mapreduce.input.ShuffledMergedInputLegacy; import org.apache.tez.mapreduce.output.SimpleOutput; import org.apache.tez.mapreduce.processor.MRTask; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java index 404dd8c..d5823f7 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java @@ -30,8 +30,6 @@ import org.apache.tez.common.records.ProceedToCompletionResponse; import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.engine.api.impl.TezHeartbeatRequest; import org.apache.tez.engine.api.impl.TezHeartbeatResponse; -import org.apache.tez.engine.records.OutputContext; -import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate; public class TestUmbilicalProtocol implements TezTaskUmbilicalProtocol { @@ -65,14 +63,6 @@ public class TestUmbilicalProtocol implements TezTaskUmbilicalProtocol { } @Override - public TezTaskDependencyCompletionEventsUpdate getDependentTasksCompletionEvents( - int fromEventIdx, int maxEventsToFetch, - TezTaskAttemptID reduce) { - // TODO Auto-generated method stub - return null; - } - - @Override public ContainerTask getTask(ContainerContext containerContext) throws IOException { // TODO Auto-generated method stub @@ -86,13 +76,6 @@ public class TestUmbilicalProtocol implements TezTaskUmbilicalProtocol { } @Override - public void outputReady(TezTaskAttemptID taskAttemptId, - OutputContext outputContext) throws IOException { - // TODO Auto-generated method stub - - } - - @Override public ProceedToCompletionResponse proceedToCompletion( TezTaskAttemptID taskAttemptId) throws IOException { return proceedToCompletionResponse; http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-yarn-client/pom.xml ---------------------------------------------------------------------- diff --git a/tez-yarn-client/pom.xml b/tez-yarn-client/pom.xml index b142701..402126e 100644 --- a/tez-yarn-client/pom.xml +++ b/tez-yarn-client/pom.xml @@ -47,7 +47,7 @@ org.apache.tez - tez-dag-api + tez-api org.apache.tez