Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 1F2A4200D29 for ; Thu, 26 Oct 2017 08:45:24 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 1D7AA160BF2; Thu, 26 Oct 2017 06:45:24 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3BFEA1609E8 for ; Thu, 26 Oct 2017 08:45:23 +0200 (CEST) Received: (qmail 97818 invoked by uid 500); 26 Oct 2017 06:45:22 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 97809 invoked by uid 99); 26 Oct 2017 06:45:21 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 26 Oct 2017 06:45:21 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D728FDFD7B; Thu, 26 Oct 2017 06:45:21 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zhiyuany@apache.org To: commits@tez.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: tez git commit: TEZ-3856. API to access counters in InputInitializerContext (Prasanth Jayachandran via zhiyuany) Date: Thu, 26 Oct 2017 06:45:21 +0000 (UTC) archived-at: Thu, 26 Oct 2017 06:45:24 -0000 Repository: tez Updated Branches: refs/heads/master d5ac3b75f -> ec9135145 TEZ-3856. API to access counters in InputInitializerContext (Prasanth Jayachandran via zhiyuany) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ec913514 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ec913514 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ec913514 Branch: refs/heads/master Commit: ec9135145fda48917b319b1accc273254c707ae5 Parents: d5ac3b7 Author: Zhiyuan Yang Authored: Wed Oct 25 23:43:46 2017 -0700 Committer: Zhiyuan Yang Committed: Wed Oct 25 23:44:02 2017 -0700 ---------------------------------------------------------------------- .../apache/tez/runtime/api/InputInitializerContext.java | 7 +++++++ .../src/main/java/org/apache/tez/dag/app/dag/Vertex.java | 6 ++++++ .../app/dag/impl/TezRootInputInitializerContextImpl.java | 5 +++++ .../java/org/apache/tez/dag/app/dag/impl/VertexImpl.java | 11 ++++++++++- .../test/java/org/apache/tez/mapreduce/TezTestUtils.java | 6 ++++++ 5 files changed, 34 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/ec913514/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java index 6a123cf..ccfac46 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java @@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.api.UserPayload; import org.apache.tez.dag.api.event.VertexState; import org.apache.tez.dag.api.event.VertexStateUpdate; @@ -117,4 +118,10 @@ public interface InputInitializerContext { */ void registerForVertexStateUpdates(String vertexName, @Nullable Set stateSet); + /** + * Add custom counters + * + * @param tezCounters counters to add + */ + void addCounters(TezCounters tezCounters); } http://git-wip-us.apache.org/repos/asf/tez/blob/ec913514/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java index 4d0a4bf..ba7624c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java @@ -86,6 +86,12 @@ public interface Vertex extends Comparable { */ TezCounters getCachedCounters(); + /** + * Add custom counters to the vertex + * @param tezCounters counters to add + */ + void addCounters(TezCounters tezCounters); + int getMaxTaskConcurrency(); Map getTasks(); Task getTask(TezTaskID taskID); http://git-wip-us.apache.org/repos/asf/tez/blob/ec913514/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java index 4ca4024..f713054 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java @@ -24,6 +24,7 @@ import java.util.Set; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.api.InputDescriptor; import org.apache.tez.dag.api.InputInitializerDescriptor; import org.apache.tez.dag.api.RootInputLeafOutput; @@ -119,4 +120,8 @@ public class TezRootInputInitializerContextImpl implements manager.registerForVertexUpdates(vertexName, input.getName(), stateSet); } + @Override + public void addCounters(final TezCounters tezCounters) { + vertex.addCounters(tezCounters); + } } http://git-wip-us.apache.org/repos/asf/tez/blob/ec913514/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java index 209db5a..0bd73ee 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java @@ -231,6 +231,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl // must be a linked map for ordering volatile LinkedHashMap tasks = new LinkedHashMap(); private Object fullCountersLock = new Object(); + private TezCounters counters = new TezCounters(); private TezCounters fullCounters = null; private TezCounters cachedCounters = null; private long cachedCountersTimestamp = 0; @@ -1189,6 +1190,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl } TezCounters counters = new TezCounters(); + counters.incrAllCounters(this.counters); return incrTaskCounters(counters, tasks.values()); } finally { @@ -1217,13 +1219,19 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl } TezCounters counters = new TezCounters(); + counters.incrAllCounters(this.counters); cachedCounters = incrTaskCounters(counters, tasks.values()); return cachedCounters; } finally { readLock.unlock(); } } - + + @Override + public void addCounters(final TezCounters tezCounters) { + counters.incrAllCounters(tezCounters); + } + @Override public int getMaxTaskConcurrency() { return vertexConf.getInt(TezConfiguration.TEZ_AM_VERTEX_MAX_TASK_CONCURRENCY, @@ -3308,6 +3316,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl @Private public void constructFinalFullcounters() { this.fullCounters = new TezCounters(); + this.fullCounters.incrAllCounters(counters); this.vertexStats = new VertexStats(); for (Task t : this.tasks.values()) { http://git-wip-us.apache.org/repos/asf/tez/blob/ec913514/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java index 8912ad2..369afbe 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java @@ -19,6 +19,7 @@ package org.apache.tez.mapreduce; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.api.UserPayload; import org.apache.tez.dag.api.event.VertexState; import org.apache.tez.dag.records.TezDAGID; @@ -120,6 +121,11 @@ public class TezTestUtils { } @Override + public void addCounters(TezCounters tezCounters) { + throw new UnsupportedOperationException("addCounters not implemented in this mock"); + } + + @Override public UserPayload getUserPayload() { throw new UnsupportedOperationException("getUserPayload not implemented in this mock"); }