tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject [1/2] git commit: TEZ-1698. Cut down on ResourceCalculatorProcessTree overheads in Tez (Rajesh Balamohan & Gopal via Rajesh Balamohan)
Date Fri, 31 Oct 2014 02:45:41 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.5 11f361268 -> 351a82513
  refs/heads/master 4ec29425d -> 4d94652c8


TEZ-1698. Cut down on ResourceCalculatorProcessTree overheads in Tez (Rajesh Balamohan &
Gopal via Rajesh Balamohan)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/4d94652c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/4d94652c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/4d94652c

Branch: refs/heads/master
Commit: 4d94652c817d83204ae06d618ade07f15fbed8d7
Parents: 4ec2942
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Fri Oct 31 08:12:21 2014 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Fri Oct 31 08:12:21 2014 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 tez-plugins/pom.xml                             | 24 ++++++
 .../tez-mbeans-resource-calculator/pom.xml      | 60 +++++++++++++++
 .../tez/util/TezMxBeanResourceCalculator.java   | 77 ++++++++++++++++++++
 .../util/TestTezMxBeanResourceCalculator.java   | 74 +++++++++++++++++++
 5 files changed, 236 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/4d94652c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 61b380e..cbbc0df 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -93,6 +93,7 @@ ALL CHANGES:
   TEZ-1722. DAG should be related to Application Id in ATS data.
   TEZ-1711. Don't cache outputSpecList in VertexImpl.getOutputSpecList(taskIndex)
   TEZ-1703. Exception handling for InputInitializer.
+  TEZ-1698. Cut down on ResourceCalculatorProcessTree overheads in Tez.
 
 Release 0.5.1: 2014-10-02
 

http://git-wip-us.apache.org/repos/asf/tez/blob/4d94652c/tez-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/tez-plugins/pom.xml b/tez-plugins/pom.xml
index b0ff489..7c15215 100644
--- a/tez-plugins/pom.xml
+++ b/tez-plugins/pom.xml
@@ -36,6 +36,30 @@
         <module>tez-yarn-timeline-history</module>
       </modules>
     </profile>
+    <profile>
+      <id>jdk6check</id>
+      <activation>
+        <property>
+          <name>java.vendor.url</name>
+          <value>http://java.sun.com/</value>
+        </property>
+      </activation>
+      <modules>
+        <module>tez-mbeans-resource-calculator</module>
+      </modules>
+    </profile>
+    <profile>
+      <id>jdk7check</id>
+      <activation>
+        <property>
+          <name>java.vendor.url</name>
+          <value>http://java.oracle.com/</value>
+        </property>
+      </activation>
+      <modules>
+        <module>tez-mbeans-resource-calculator</module>
+      </modules>
+    </profile>
   </profiles>
 
   <build>

http://git-wip-us.apache.org/repos/asf/tez/blob/4d94652c/tez-plugins/tez-mbeans-resource-calculator/pom.xml
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-mbeans-resource-calculator/pom.xml b/tez-plugins/tez-mbeans-resource-calculator/pom.xml
new file mode 100644
index 0000000..bb1a0ce
--- /dev/null
+++ b/tez-plugins/tez-mbeans-resource-calculator/pom.xml
@@ -0,0 +1,60 @@
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.tez</groupId>
+    <artifactId>tez-plugins</artifactId>
+    <version>0.6.0-SNAPSHOT</version>
+  </parent>
+  <artifactId>tez-mbeans-resource-calculator</artifactId>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-dag</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

http://git-wip-us.apache.org/repos/asf/tez/blob/4d94652c/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
b/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
new file mode 100644
index 0000000..e8aeb0b
--- /dev/null
+++ b/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
@@ -0,0 +1,77 @@
+package org.apache.tez.util;
+
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+
+import java.lang.management.ManagementFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Uses sun's MBeans to return process information.
+ */
+public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree {
+
+  private final com.sun.management.OperatingSystemMXBean osBean;
+  private final Runtime runtime;
+  private final AtomicLong cumulativeCPU;
+
+  /**
+   * Create process-tree instance with specified root process.
+   * <p/>
+   * Subclass must override this.
+   *
+   * @param root process-tree root-process
+   */
+  public TezMxBeanResourceCalculator(String root) {
+    super(root);
+    runtime = Runtime.getRuntime();
+    osBean =
+        (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
+    cumulativeCPU = new AtomicLong();
+  }
+
+  @Override public void updateProcessTree() {
+    //nothing needs to be done as the data is read from OS mbeans.
+  }
+
+  @Override public String getProcessTreeDump() {
+    return "";
+  }
+
+  @Override public long getCumulativeVmem(int olderThanAge) {
+    return osBean.getCommittedVirtualMemorySize();
+  }
+
+  @Override public long getCumulativeRssmem(int olderThanAge) {
+    //Not supported directly (RSS ~= memory consumed by JVM from Xmx)
+    return runtime.totalMemory();
+  }
+
+  @Override public long getCumulativeCpuTime() {
+    //convert to milliseconds
+    return TimeUnit.MILLISECONDS.convert(cumulativeCPU.addAndGet(osBean.getProcessCpuTime()),
+        TimeUnit.MILLISECONDS);
+  }
+
+  @Override public boolean checkPidPgrpidForMatch() {
+    return true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/4d94652c/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java
b/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java
new file mode 100644
index 0000000..ade2118
--- /dev/null
+++ b/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java
@@ -0,0 +1,74 @@
+package org.apache.tez.util;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.history.DAGHistoryEvent;
+import org.apache.tez.dag.history.events.DAGStartedEvent;
+import org.apache.tez.dag.records.TezDAGID;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class TestTezMxBeanResourceCalculator {
+
+  private ResourceCalculatorProcessTree resourceCalculator;
+
+  @Before
+  public void setup() throws Exception {
+    Configuration conf = new TezConfiguration();
+    conf.set(TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS,
+        TezMxBeanResourceCalculator.class.getName());
+
+    Class<? extends ResourceCalculatorProcessTree> clazz = conf.getClass(
+        TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null,
+        ResourceCalculatorProcessTree.class);
+    resourceCalculator = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(
+        System.getenv().get("JVM_PID"), clazz, conf);
+  }
+
+  @After
+  public void teardown() {
+    resourceCalculator = null;
+  }
+
+  @Test(timeout=5000)
+  public void testResourceCalculator() {
+    Assert.assertTrue(resourceCalculator instanceof TezMxBeanResourceCalculator);
+    Assert.assertTrue(resourceCalculator.getCumulativeCpuTime() > 0);
+    Assert.assertTrue(resourceCalculator.getCumulativeVmem() > 0);
+    Assert.assertTrue(resourceCalculator.getCumulativeRssmem() > 0);
+    Assert.assertTrue(resourceCalculator.getProcessTreeDump().equals(""));
+    Assert.assertTrue(resourceCalculator.checkPidPgrpidForMatch());
+  }
+
+}


Mime
View raw message