myriad-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dar...@apache.org
Subject incubator-myriad git commit: MYRIAD-218 Cleaned up TaskUtils. Removed unused methods and methods which are only used in unit tests JIRA: [MYRIAD-218] https://issues.apache.org/jira/browse/MYRIAD-218 Pull Request: Closes #87 Author: ponnu <ponnuling
Date Fri, 29 Jul 2016 20:36:35 GMT
Repository: incubator-myriad
Updated Branches:
  refs/heads/master dcccf2e3b -> df4cbc0e9


MYRIAD-218 Cleaned up TaskUtils. Removed unused methods and methods
which are only used in unit tests
JIRA:
	[MYRIAD-218] https://issues.apache.org/jira/browse/MYRIAD-218
Pull Request:
  Closes #87
Author:    ponnu <ponnulingam.r@gmail.com>
Date:      Mon Jul 25 21:03:15 2016 -0400


Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/df4cbc0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/df4cbc0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/df4cbc0e

Branch: refs/heads/master
Commit: df4cbc0e97024cff4064e58fa792b3cbf3eee4f3
Parents: dcccf2e
Author: ponnu <ponnulingam.r@gmail.com>
Authored: Mon Jul 25 21:03:15 2016 -0400
Committer: darinj <darinj@apache.org>
Committed: Fri Jul 29 16:33:39 2016 -0400

----------------------------------------------------------------------
 .../org/apache/myriad/scheduler/TaskUtils.java  | 162 +------------------
 .../apache/myriad/scheduler/TestTaskUtils.java  |  33 +---
 2 files changed, 13 insertions(+), 182 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/df4cbc0e/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
index c8e2a21..6ab6ee0 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
@@ -18,60 +18,28 @@
  */
 package org.apache.myriad.scheduler;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerConfigurationException;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-import javax.xml.xpath.XPath;
-import javax.xml.xpath.XPathConstants;
-import javax.xml.xpath.XPathExpression;
-import javax.xml.xpath.XPathExpressionException;
-import javax.xml.xpath.XPathFactory;
-
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
 import org.apache.mesos.Protos;
-import org.apache.myriad.configuration.MyriadBadConfigurationException;
 import org.apache.myriad.configuration.MyriadConfiguration;
 import org.apache.myriad.configuration.MyriadContainerConfiguration;
 import org.apache.myriad.configuration.MyriadDockerConfiguration;
-import org.apache.myriad.configuration.NodeManagerConfiguration;
-import org.apache.myriad.configuration.ServiceConfiguration;
 import org.apache.myriad.executor.MyriadExecutorDefaults;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
 
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 
 /**
  * utility class for working with tasks and node manager profiles
  */
 public class TaskUtils {
   private static final Logger LOGGER = LoggerFactory.getLogger(TaskUtils.class);
-  private static final String YARN_NODEMANAGER_RESOURCE_CPU_VCORES = "yarn.nodemanager.resource.cpu-vcores";
-  private static final String YARN_NODEMANAGER_RESOURCE_MEMORY_MB = "yarn.nodemanager.resource.memory-mb";
   private static final String CONTAINER_PATH_KEY = "containerPath";
   private static final String HOST_PATH_KEY = "hostPath";
   private static final String RW_MODE = "mode";
@@ -79,90 +47,12 @@ public class TaskUtils {
   private static final String PARAMETER_VALUE_KEY = "value";
 
   private MyriadConfiguration cfg;
-  Random random = new Random();
 
   @Inject
   public TaskUtils(MyriadConfiguration cfg) {
     this.cfg = cfg;
   }
 
-  public static String getRevisedConfig(Double cpu, Double memory) {
-    String revisedConfig = "";
-    try {
-
-      // todo:(kgs) replace with more abstract xml parser
-      DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
-      factory.setNamespaceAware(true);
-      DocumentBuilder builder;
-      Document doc;
-
-      builder = factory.newDocumentBuilder();
-      InputStream resourceAsStream = TaskUtils.class.getClassLoader().getResourceAsStream("yarn-site.xml");
-
-      doc = builder.parse(new InputSource(resourceAsStream));
-      resourceAsStream.close();
-
-      XPathFactory xFactory = XPathFactory.newInstance();
-
-      XPath xpath = xFactory.newXPath();
-      XPathExpression cpuXpath = xpath.compile("//property/name");
-      Object cpuNodeObj = cpuXpath.evaluate(doc, XPathConstants.NODESET);
-
-      NodeList cpuNode = (NodeList) cpuNodeObj;
-
-      for (int i = 0; i < cpuNode.getLength(); i++) {
-        Node item = cpuNode.item(i);
-        if (YARN_NODEMANAGER_RESOURCE_CPU_VCORES.equals(item.getTextContent())) {
-          Node propertyNode = item.getParentNode();
-          NodeList childNodes = propertyNode.getChildNodes();
-          for (int j = 0; j < childNodes.getLength(); j++) {
-            Node item2 = childNodes.item(j);
-            if ("value".equals(item2.getNodeName())) {
-              item2.setTextContent(cpu.intValue() + "");
-            }
-          }
-        } else if (YARN_NODEMANAGER_RESOURCE_MEMORY_MB.equals(item.getTextContent())) {
-          Node propertyNode = item.getParentNode();
-          NodeList childNodes = propertyNode.getChildNodes();
-          for (int j = 0; j < childNodes.getLength(); j++) {
-            Node item2 = childNodes.item(j);
-            if ("value".equals(item2.getNodeName())) {
-              item2.setTextContent(memory.intValue() + "");
-            }
-          }
-        }
-      }
-
-      TransformerFactory tf = TransformerFactory.newInstance();
-      Transformer transformer = tf.newTransformer();
-      transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
-      StringWriter writer = new StringWriter();
-      transformer.transform(new DOMSource(doc), new StreamResult(writer));
-
-      revisedConfig = writer.getBuffer().toString().replaceAll("\n|\r", "");
-    } catch (TransformerConfigurationException e) {
-      e.printStackTrace();
-    } catch (TransformerException | SAXException | XPathExpressionException | ParserConfigurationException
e) {
-      LOGGER.error("Error with xml operation", e);
-    } catch (IOException e) {
-      LOGGER.error("Error with xml operation", e);
-    }
-    return revisedConfig;
-  }
-
-  public double getAggregateMemory(NMProfile profile) {
-    double totalTaskMemory;
-    double executorMemory = getExecutorMemory();
-    double nmJvmMaxMemoryMB = getNodeManagerMemory();
-    double advertisableMemory = profile.getMemory();
-    totalTaskMemory = executorMemory + nmJvmMaxMemoryMB + advertisableMemory;
-    return totalTaskMemory;
-  }
-
-  public double getAggregateCpus(NMProfile profile) {
-    return getNodeManagerCpus() + MyriadExecutorDefaults.DEFAULT_CPUS + profile.getCpus();
-  }
-
   public double getNodeManagerMemory() {
     return cfg.getNodeManagerConfiguration().getJvmMaxMemoryMB();
   }
@@ -180,42 +70,6 @@ public class TaskUtils {
     return cfg.getMyriadExecutorConfiguration().getJvmMaxMemoryMB();
   }
 
-  public double getTaskCpus(NMProfile profile) {
-
-    return getAggregateCpus(profile) - getExecutorCpus();
-  }
-
-  public double getTaskMemory(NMProfile profile) {
-
-    return getAggregateMemory(profile) - getExecutorMemory();
-  }
-
-  public double getAuxTaskCpus(NMProfile profile, String taskName) throws MyriadBadConfigurationException
{
-    if (taskName.startsWith(NodeManagerConfiguration.NM_TASK_PREFIX)) {
-      return getAggregateCpus(profile);
-    }
-
-    Optional<ServiceConfiguration> auxConf = cfg.getServiceConfiguration(taskName);
-    if (!auxConf.isPresent()) {
-      throw new MyriadBadConfigurationException("Can not find profile for task name: " +
taskName);
-    }
-
-    return auxConf.get().getCpus();
-  }
-
-  public double getAuxTaskMemory(NMProfile profile, String taskName) throws MyriadBadConfigurationException
{
-    if (taskName.startsWith(NodeManagerConfiguration.NM_TASK_PREFIX)) {
-      return getAggregateMemory(profile);
-    }
-  
-    Optional<ServiceConfiguration> auxConf = cfg.getServiceConfiguration(taskName);
   
-    if (!auxConf.isPresent()) {
-      throw new MyriadBadConfigurationException("Cannot find profile for task name: " + taskName);
-    }
-
-    return auxConf.get().getJvmMaxMemoryMB();
-  }
-
   public TaskUtils() {
     super();
   }

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/df4cbc0e/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
index 29d7691..d543c29 100644
--- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
@@ -17,18 +17,15 @@
  */
 package org.apache.myriad.scheduler;
 
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.apache.mesos.Protos;
-import org.apache.myriad.BaseConfigurableTest;
-import org.apache.myriad.configuration.MyriadBadConfigurationException;
-import org.junit.Test;
-
 import com.google.common.collect.Range;
 import com.google.common.collect.Ranges;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import org.apache.mesos.Protos;
+import org.apache.myriad.BaseConfigurableTest;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
 
 /**
  * Tests for TaskUtils
@@ -37,26 +34,6 @@ public class TestTaskUtils extends BaseConfigurableTest {
   static double epsilon = .0001;
 
   @Test
-  public void testGetResource() {
-    TaskUtils taskUtils = new TaskUtils(cfg);
-
-    NMProfile fooProfile = new NMProfile("abc", 1L, 1000L);
-    try {
-      taskUtils.getAuxTaskCpus(fooProfile, "foo");
-      fail("Should not complete sucessfully for foo");
-    } catch (MyriadBadConfigurationException e) {
-      // success
-    }
-
-    try {
-      double cpu = taskUtils.getAuxTaskCpus(fooProfile, "jobhistory");
-      assertTrue(cpu > 0.0);
-    } catch (MyriadBadConfigurationException e) {
-      fail("cpu should be defined for jobhistory");
-    }
-  }
-
-  @Test
   public void testServiceResourceProfile() throws Exception {
     // testing custom deserializer
 


Mime
View raw message