apex-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vro...@apache.org
Subject apex-core git commit: APEXCORE-736 Using YARN client api to fetch the application master container report, this closes #534
Date Fri, 09 Jun 2017 02:00:16 GMT
Repository: apex-core
Updated Branches:
  refs/heads/master a0dd30d8f -> 91eb45078


APEXCORE-736 Using YARN client api to fetch the application master container report, this
closes #534


Project: http://git-wip-us.apache.org/repos/asf/apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-core/commit/91eb4507
Tree: http://git-wip-us.apache.org/repos/asf/apex-core/tree/91eb4507
Diff: http://git-wip-us.apache.org/repos/asf/apex-core/diff/91eb4507

Branch: refs/heads/master
Commit: 91eb45078b27d27dfcda35d124dce90f57f39efb
Parents: a0dd30d
Author: devtagare <devtagare@gmail.com>
Authored: Wed May 31 14:09:00 2017 -0700
Committer: Pramod Immaneni <pramod@datatorrent.com>
Committed: Thu Jun 8 18:55:47 2017 -0700

----------------------------------------------------------------------
 .../datatorrent/stram/StreamingContainerManager.java | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/apex-core/blob/91eb4507/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java b/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
index c4e76a5..510a146 100644
--- a/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
+++ b/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
@@ -84,6 +84,9 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.SystemClock;
@@ -168,7 +171,6 @@ import com.datatorrent.stram.util.ConfigUtils;
 import com.datatorrent.stram.util.FSJsonLineFile;
 import com.datatorrent.stram.util.MovingAverage.MovingAverageLong;
 import com.datatorrent.stram.util.SharedPubSubWebSocketClient;
-import com.datatorrent.stram.util.WebServicesClient;
 import com.datatorrent.stram.webapp.ContainerInfo;
 import com.datatorrent.stram.webapp.LogicalOperatorInfo;
 import com.datatorrent.stram.webapp.OperatorAggregationInfo;
@@ -485,11 +487,12 @@ public class StreamingContainerManager implements PlanContext
         String nodeHttpAddress = nmHost + ":" + nmHttpPort;
         if (allocatedMemoryMB == 0) {
           String url = ConfigUtils.getSchemePrefix(conf) + nodeHttpAddress + "/ws/v1/node/containers/"
+ ci.id;
-          WebServicesClient webServicesClient = new WebServicesClient();
-          try {
-            String content = webServicesClient.process(url, String.class, new WebServicesClient.GetWebServicesHandler<String>());
-            JSONObject json = new JSONObject(content);
-            int totalMemoryNeededMB = json.getJSONObject("container").getInt("totalMemoryNeededMB");
+          try (YarnClient rmClient = YarnClient.createYarnClient()) {
+            rmClient.init(conf);
+            rmClient.start();
+            ContainerReport content = rmClient.getContainerReport(ContainerId.fromString(ci.id));
+            int totalMemoryNeededMB = content.getAllocatedResource().getMemory();
+            LOG.debug("App Master allocated memory is {}", totalMemoryNeededMB);
             if (totalMemoryNeededMB > 0) {
               allocatedMemoryMB = totalMemoryNeededMB;
             } else {


Mime
View raw message