ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mpapirkovs...@apache.org
Subject ambari git commit: AMBARI-14519. GET calls to /requests become very slow after installing packages on 900 node cluster. (mpapirkovskyy)
Date Tue, 29 Dec 2015 17:46:17 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.2 af0e2ade6 -> a9b3c6c04


AMBARI-14519. GET calls to /requests become very slow after installing packages on 900 node
cluster. (mpapirkovskyy)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a9b3c6c0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a9b3c6c0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a9b3c6c0

Branch: refs/heads/branch-2.2
Commit: a9b3c6c0450f24381191e2b199a25dbe6220646d
Parents: af0e2ad
Author: Myroslav Papirkovskyy <mpapyrkovskyy@hortonworks.com>
Authored: Tue Dec 29 17:23:58 2015 +0200
Committer: Myroslav Papirkovskyy <mpapyrkovskyy@hortonworks.com>
Committed: Tue Dec 29 19:45:56 2015 +0200

----------------------------------------------------------------------
 .../ambari/server/topology/AmbariContext.java   |  4 +++
 .../ambari/server/topology/HostRequest.java     |  4 +++
 .../ambari/server/topology/LogicalRequest.java  | 35 +++++++++++++++++---
 .../server/upgrade/UpgradeCatalog221.java       |  6 +++-
 .../main/resources/Ambari-DDL-MySQL-CREATE.sql  |  5 +++
 .../main/resources/Ambari-DDL-Oracle-CREATE.sql |  5 +++
 .../resources/Ambari-DDL-Postgres-CREATE.sql    |  5 +++
 .../Ambari-DDL-Postgres-EMBEDDED-CREATE.sql     |  5 +++
 .../resources/Ambari-DDL-SQLAnywhere-CREATE.sql |  5 +++
 .../resources/Ambari-DDL-SQLServer-CREATE.sql   |  5 +++
 .../server/upgrade/UpgradeCatalog221Test.java   | 31 +++++++++++++++++
 11 files changed, 105 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/a9b3c6c0/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index d9ac183..4ccda44 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -144,6 +144,10 @@ public class AmbariContext {
     return getController().getActionManager().getTaskById(id);
   }
 
+  public Collection<HostRoleCommand> getPhysicalTasks(Collection<Long> ids) {
+    return getController().getActionManager().getTasks(ids);
+  }
+
   public void createAmbariResources(ClusterTopology topology, String clusterName, SecurityType
securityType) {
     Stack stack = topology.getBlueprint().getStack();
     createAmbariClusterResource(clusterName, stack.getName(), stack.getVersion(), securityType);

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9b3c6c0/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
index 7f178f4..440638c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
@@ -387,6 +387,10 @@ public class HostRequest implements Comparable<HostRequest> {
     return physicalTasks.get(logicalTaskId);
   }
 
+  public Map<Long, Long> getPhysicalTaskMapping() {
+    return new HashMap<>(physicalTasks);
+  }
+
   //todo: since this is used to determine equality, using hashCode() isn't safe as it can
return the same
   //todo: value for 2 unequal requests
   @Override

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9b3c6c0/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
index fb6b774..bd9f2e0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
@@ -195,6 +195,7 @@ public class LogicalRequest extends Request {
   }
 
   // currently we are just returning all stages for all requests
+  //TODO technically StageEntity is simply a container for HostRequest info with additional
redundant transformations
   public Collection<StageEntity> getStageEntities() {
     Collection<StageEntity> stages = new ArrayList<StageEntity>();
     for (HostRequest hostRequest : allHostRequests) {
@@ -232,7 +233,32 @@ public class LogicalRequest extends Request {
   public Map<Long, HostRoleCommandStatusSummaryDTO> getStageSummaries() {
     Map<Long, HostRoleCommandStatusSummaryDTO> summaryMap = new HashMap<Long, HostRoleCommandStatusSummaryDTO>();
 
-    for (StageEntity stage : getStageEntities()) {
+    Map<Long, Collection<HostRoleCommand>> stageTasksMap = new HashMap<>();
+
+    Map<Long, Long> taskToStageMap = new HashMap<>();
+
+
+    for (HostRequest hostRequest : getHostRequests()) {
+      Map<Long, Long> physicalTaskMapping = hostRequest.getPhysicalTaskMapping();
+      Collection<Long> stageTasks = physicalTaskMapping.values();
+      for (Long stageTask : stageTasks) {
+        taskToStageMap.put(stageTask, hostRequest.getStageId());
+      }
+    }
+
+    Collection<HostRoleCommand> physicalTasks = topology.getAmbariContext().getPhysicalTasks(taskToStageMap.keySet());
+
+    for (HostRoleCommand physicalTask : physicalTasks) {
+      Long stageId = taskToStageMap.get(physicalTask.getTaskId());
+      Collection<HostRoleCommand> stageTasks = stageTasksMap.get(stageId);
+      if (stageTasks == null) {
+        stageTasks = new ArrayList<>();
+        stageTasksMap.put(stageId, stageTasks);
+      }
+      stageTasks.add(physicalTask);
+    }
+
+    for (Long stageId : stageTasksMap.keySet()) {
       //Number minStartTime = 0;
       //Number maxEndTime = 0;
       int aborted = 0;
@@ -248,7 +274,7 @@ public class LogicalRequest extends Request {
       int skippedFailed = 0;
 
       //todo: where does this logic belong?
-      for (HostRoleCommandEntity task : stage.getHostRoleCommands()) {
+      for (HostRoleCommand task : stageTasksMap.get(stageId)) {
         HostRoleStatus taskStatus = task.getStatus();
 
         switch (taskStatus) {
@@ -291,12 +317,13 @@ public class LogicalRequest extends Request {
       }
 
       HostRoleCommandStatusSummaryDTO stageSummary = new HostRoleCommandStatusSummaryDTO(
-          stage.isSkippable() ? 1 : 0, 0, 0, stage.getStageId(), aborted, completed, failed,
+          0, 0, 0, stageId, aborted, completed, failed,
           holding, holdingFailed, holdingTimedout, inProgress, pending, queued, timedout,
           skippedFailed);
 
-      summaryMap.put(stage.getStageId(), stageSummary);
+      summaryMap.put(stageId, stageSummary);
     }
+
     return summaryMap;
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9b3c6c0/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
index b33d44e..c4216bd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
@@ -109,7 +109,11 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog {
 
   @Override
   protected void executeDDLUpdates() throws AmbariException, SQLException {
-    //To change body of implemented methods use File | Settings | File Templates.
+    // indices to improve request status calc performance
+    dbAccessor.createIndex("idx_stage_request_id", "stage", "request_id");
+    dbAccessor.createIndex("idx_hrc_request_id", "host_role_command", "request_id");
+    dbAccessor.createIndex("idx_rsc_request_id", "role_success_criteria", "request_id");
+
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9b3c6c0/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
index c00e46f..e217547 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
@@ -648,6 +648,11 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY (id)
 );
 
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
 -- altering tables by creating unique constraints----------
 ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, ldap_user);
 ALTER TABLE groups ADD CONSTRAINT UNQ_groups_0 UNIQUE (group_name, ldap_group);

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9b3c6c0/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
index 3cac618..654b420 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
@@ -637,6 +637,11 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY (id)
 );
 
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
 --------altering tables by creating unique constraints----------
 ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, ldap_user);
 ALTER TABLE groups ADD CONSTRAINT UNQ_groups_0 UNIQUE (group_name, ldap_group);

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9b3c6c0/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
index 2bcfb9a..75bd171 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
@@ -642,6 +642,11 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY (id)
 );
 
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
 --------altering tables by creating unique constraints----------
 ALTER TABLE clusterconfig ADD CONSTRAINT UQ_config_type_tag UNIQUE (cluster_id, type_name,
version_tag);
 ALTER TABLE clusterconfig ADD CONSTRAINT UQ_config_type_version UNIQUE (cluster_id, type_name,
version);

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9b3c6c0/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
index 56dff9b..e9464ae 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
@@ -721,6 +721,11 @@ CREATE TABLE ambari.topology_logical_task (
 );
 GRANT ALL PRIVILEGES ON TABLE ambari.topology_logical_task TO :username;
 
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON ambari.stage (request_id);
+CREATE INDEX idx_hrc_request_id ON ambari.host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON ambari.role_success_criteria (request_id);
+
 --------altering tables by creating unique constraints----------
 ALTER TABLE ambari.clusterconfig ADD CONSTRAINT UQ_config_type_tag UNIQUE (cluster_id, type_name,
version_tag);
 ALTER TABLE ambari.clusterconfig ADD CONSTRAINT UQ_config_type_version UNIQUE (cluster_id,
type_name, version);

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9b3c6c0/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
index 1cc83ed..52994d6 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
@@ -638,6 +638,11 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY (id)
 );
 
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
 -- altering tables by creating unique constraints----------
 ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, ldap_user);
 ALTER TABLE groups ADD CONSTRAINT UNQ_groups_0 UNIQUE (group_name, ldap_group);

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9b3c6c0/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
index 1600ca2..7f01df3 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
@@ -750,6 +750,11 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY CLUSTERED (id)
 );
 
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
 
 -- altering tables by creating unique constraints----------
 --------altering tables to add constraints----------

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9b3c6c0/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
index bfe26e7..23d9a70 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
@@ -22,8 +22,10 @@ package org.apache.ambari.server.upgrade;
 import com.google.inject.AbstractModule;
 import com.google.common.collect.Maps;
 import com.google.gson.Gson;
+import com.google.inject.Binder;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
+import com.google.inject.Module;
 import com.google.inject.Provider;
 import com.google.inject.persist.PersistService;
 import junit.framework.Assert;
@@ -62,6 +64,7 @@ import java.util.Map;
 import static org.easymock.EasyMock.createMockBuilder;
 import static org.easymock.EasyMock.createNiceMock;
 import static org.easymock.EasyMock.createStrictMock;
+import static org.easymock.EasyMock.eq;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.expectLastCall;
 import static org.easymock.EasyMock.replay;
@@ -101,6 +104,34 @@ public class UpgradeCatalog221Test {
   }
 
   @Test
+  public void testExecuteDDLUpdates() throws Exception{
+    final DBAccessor dbAccessor = createNiceMock(DBAccessor.class);
+
+    dbAccessor.createIndex(eq("idx_stage_request_id"), eq("stage"), eq("request_id"));
+    expectLastCall().once();
+    dbAccessor.createIndex(eq("idx_hrc_request_id"), eq("host_role_command"), eq("request_id"));
+    expectLastCall().once();
+    dbAccessor.createIndex(eq("idx_rsc_request_id"), eq("role_success_criteria"), eq("request_id"));
+    expectLastCall().once();
+
+
+    replay(dbAccessor);
+    Module module = new Module() {
+      @Override
+      public void configure(Binder binder) {
+        binder.bind(DBAccessor.class).toInstance(dbAccessor);
+        binder.bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
+        binder.bind(EntityManager.class).toInstance(entityManager);
+      }
+    };
+
+    Injector injector = Guice.createInjector(module);
+    UpgradeCatalog221 upgradeCatalog221 = injector.getInstance(UpgradeCatalog221.class);
+    upgradeCatalog221.executeDDLUpdates();
+    verify(dbAccessor);
+  }
+
+  @Test
   public void testExecuteDMLUpdates() throws Exception {
     Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml");
     Method updateAlerts = UpgradeCatalog221.class.getDeclaredMethod("updateAlerts");


Mime
View raw message