ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [26/50] [abbrv] ambari git commit: AMBARI-14519. GET calls to /requests become very slow after installing packages on 900 node cluster. (mpapirkovskyy)
Date Mon, 04 Jan 2016 21:38:19 GMT
AMBARI-14519. GET calls to /requests become very slow after installing packages on 900 node
cluster. (mpapirkovskyy)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8a748ffd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8a748ffd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8a748ffd

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 8a748ffd9829cc25350ea79ed9f44abe0a98b012
Parents: 9022bfb
Author: Myroslav Papirkovskyy <mpapyrkovskyy@hortonworks.com>
Authored: Tue Dec 29 16:45:15 2015 +0200
Committer: Myroslav Papirkovskyy <mpapyrkovskyy@hortonworks.com>
Committed: Tue Dec 29 19:45:27 2015 +0200

----------------------------------------------------------------------
 .../ambari/server/topology/AmbariContext.java   |  4 +++
 .../ambari/server/topology/HostRequest.java     |  4 +++
 .../ambari/server/topology/LogicalRequest.java  | 35 +++++++++++++++++---
 .../server/upgrade/UpgradeCatalog221.java       |  6 +++-
 .../main/resources/Ambari-DDL-Derby-CREATE.sql  |  5 +++
 .../main/resources/Ambari-DDL-MySQL-CREATE.sql  |  5 +++
 .../main/resources/Ambari-DDL-Oracle-CREATE.sql |  5 +++
 .../resources/Ambari-DDL-Postgres-CREATE.sql    |  5 +++
 .../Ambari-DDL-Postgres-EMBEDDED-CREATE.sql     |  5 +++
 .../resources/Ambari-DDL-SQLAnywhere-CREATE.sql |  5 +++
 .../resources/Ambari-DDL-SQLServer-CREATE.sql   |  5 +++
 .../server/upgrade/UpgradeCatalog221Test.java   | 31 +++++++++++++++++
 12 files changed, 110 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index 0a2bd18..6bfee93 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -145,6 +145,10 @@ public class AmbariContext {
     return getController().getActionManager().getTaskById(id);
   }
 
+  public Collection<HostRoleCommand> getPhysicalTasks(Collection<Long> ids) {
+    return getController().getActionManager().getTasks(ids);
+  }
+
   public void createAmbariResources(ClusterTopology topology, String clusterName, SecurityType
securityType) {
     Stack stack = topology.getBlueprint().getStack();
     createAmbariClusterResource(clusterName, stack.getName(), stack.getVersion(), securityType);

http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
index 7f178f4..440638c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
@@ -387,6 +387,10 @@ public class HostRequest implements Comparable<HostRequest> {
     return physicalTasks.get(logicalTaskId);
   }
 
+  public Map<Long, Long> getPhysicalTaskMapping() {
+    return new HashMap<>(physicalTasks);
+  }
+
   //todo: since this is used to determine equality, using hashCode() isn't safe as it can
return the same
   //todo: value for 2 unequal requests
   @Override

http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
index fb6b774..bd9f2e0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
@@ -195,6 +195,7 @@ public class LogicalRequest extends Request {
   }
 
   // currently we are just returning all stages for all requests
+  //TODO technically StageEntity is simply a container for HostRequest info with additional
redundant transformations
   public Collection<StageEntity> getStageEntities() {
     Collection<StageEntity> stages = new ArrayList<StageEntity>();
     for (HostRequest hostRequest : allHostRequests) {
@@ -232,7 +233,32 @@ public class LogicalRequest extends Request {
   public Map<Long, HostRoleCommandStatusSummaryDTO> getStageSummaries() {
     Map<Long, HostRoleCommandStatusSummaryDTO> summaryMap = new HashMap<Long, HostRoleCommandStatusSummaryDTO>();
 
-    for (StageEntity stage : getStageEntities()) {
+    Map<Long, Collection<HostRoleCommand>> stageTasksMap = new HashMap<>();
+
+    Map<Long, Long> taskToStageMap = new HashMap<>();
+
+
+    for (HostRequest hostRequest : getHostRequests()) {
+      Map<Long, Long> physicalTaskMapping = hostRequest.getPhysicalTaskMapping();
+      Collection<Long> stageTasks = physicalTaskMapping.values();
+      for (Long stageTask : stageTasks) {
+        taskToStageMap.put(stageTask, hostRequest.getStageId());
+      }
+    }
+
+    Collection<HostRoleCommand> physicalTasks = topology.getAmbariContext().getPhysicalTasks(taskToStageMap.keySet());
+
+    for (HostRoleCommand physicalTask : physicalTasks) {
+      Long stageId = taskToStageMap.get(physicalTask.getTaskId());
+      Collection<HostRoleCommand> stageTasks = stageTasksMap.get(stageId);
+      if (stageTasks == null) {
+        stageTasks = new ArrayList<>();
+        stageTasksMap.put(stageId, stageTasks);
+      }
+      stageTasks.add(physicalTask);
+    }
+
+    for (Long stageId : stageTasksMap.keySet()) {
       //Number minStartTime = 0;
       //Number maxEndTime = 0;
       int aborted = 0;
@@ -248,7 +274,7 @@ public class LogicalRequest extends Request {
       int skippedFailed = 0;
 
       //todo: where does this logic belong?
-      for (HostRoleCommandEntity task : stage.getHostRoleCommands()) {
+      for (HostRoleCommand task : stageTasksMap.get(stageId)) {
         HostRoleStatus taskStatus = task.getStatus();
 
         switch (taskStatus) {
@@ -291,12 +317,13 @@ public class LogicalRequest extends Request {
       }
 
       HostRoleCommandStatusSummaryDTO stageSummary = new HostRoleCommandStatusSummaryDTO(
-          stage.isSkippable() ? 1 : 0, 0, 0, stage.getStageId(), aborted, completed, failed,
+          0, 0, 0, stageId, aborted, completed, failed,
           holding, holdingFailed, holdingTimedout, inProgress, pending, queued, timedout,
           skippedFailed);
 
-      summaryMap.put(stage.getStageId(), stageSummary);
+      summaryMap.put(stageId, stageSummary);
     }
+
     return summaryMap;
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
index b33d44e..c4216bd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
@@ -109,7 +109,11 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog {
 
   @Override
   protected void executeDDLUpdates() throws AmbariException, SQLException {
-    //To change body of implemented methods use File | Settings | File Templates.
+    // indices to improve request status calc performance
+    dbAccessor.createIndex("idx_stage_request_id", "stage", "request_id");
+    dbAccessor.createIndex("idx_hrc_request_id", "host_role_command", "request_id");
+    dbAccessor.createIndex("idx_rsc_request_id", "role_success_criteria", "request_id");
+
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
index 0e534b7..d93a3c2 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
@@ -653,6 +653,11 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY (id)
 );
 
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
 --------altering tables by creating unique constraints----------
 ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type);
 ALTER TABLE clusterconfig ADD CONSTRAINT UQ_config_type_tag UNIQUE (version_tag, type_name,
cluster_id);

http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
index fcfcb93..aa8ced1 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
@@ -660,6 +660,11 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY (id)
 );
 
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
 -- altering tables by creating unique constraints----------
 ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type);
 ALTER TABLE groups ADD CONSTRAINT UNQ_groups_0 UNIQUE (group_name, ldap_group);

http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
index 4305a4c..b534344 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
@@ -649,6 +649,11 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY (id)
 );
 
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
 --------altering tables by creating unique constraints----------
 ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type);
 ALTER TABLE groups ADD CONSTRAINT UNQ_groups_0 UNIQUE (group_name, ldap_group);

http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
index 32fd2a6..941fc6e 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
@@ -653,6 +653,11 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY (id)
 );
 
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
 --------altering tables by creating unique constraints----------
 ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type);
 ALTER TABLE clusterconfig ADD CONSTRAINT UQ_config_type_tag UNIQUE (cluster_id, type_name,
version_tag);

http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
index 7894c43..dd517f8 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
@@ -734,6 +734,11 @@ CREATE TABLE ambari.topology_logical_task (
 );
 GRANT ALL PRIVILEGES ON TABLE ambari.topology_logical_task TO :username;
 
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON ambari.stage (request_id);
+CREATE INDEX idx_hrc_request_id ON ambari.host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON ambari.role_success_criteria (request_id);
+
 --------altering tables by creating unique constraints----------
 ALTER TABLE ambari.users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type);
 ALTER TABLE ambari.clusterconfig ADD CONSTRAINT UQ_config_type_tag UNIQUE (cluster_id, type_name,
version_tag);

http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
index 6df1569..f837f9e 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
@@ -650,6 +650,11 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY (id)
 );
 
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
 -- altering tables by creating unique constraints----------
 ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type);
 ALTER TABLE groups ADD CONSTRAINT UNQ_groups_0 UNIQUE (group_name, ldap_group);

http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
index 83b2a7e..239d27e 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
@@ -758,6 +758,11 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY CLUSTERED (id)
 );
 
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
 
 -- altering tables by creating unique constraints----------
 --------altering tables to add constraints----------

http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
index bbae7a7..5bd8f2a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
@@ -22,8 +22,10 @@ package org.apache.ambari.server.upgrade;
 import com.google.inject.AbstractModule;
 import com.google.common.collect.Maps;
 import com.google.gson.Gson;
+import com.google.inject.Binder;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
+import com.google.inject.Module;
 import com.google.inject.Provider;
 import com.google.inject.persist.PersistService;
 import junit.framework.Assert;
@@ -60,6 +62,7 @@ import java.util.Map;
 import static org.easymock.EasyMock.createMockBuilder;
 import static org.easymock.EasyMock.createNiceMock;
 import static org.easymock.EasyMock.createStrictMock;
+import static org.easymock.EasyMock.eq;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.expectLastCall;
 import static org.easymock.EasyMock.replay;
@@ -101,6 +104,34 @@ public class UpgradeCatalog221Test {
   }
 
   @Test
+  public void testExecuteDDLUpdates() throws Exception{
+    final DBAccessor dbAccessor = createNiceMock(DBAccessor.class);
+
+    dbAccessor.createIndex(eq("idx_stage_request_id"), eq("stage"), eq("request_id"));
+    expectLastCall().once();
+    dbAccessor.createIndex(eq("idx_hrc_request_id"), eq("host_role_command"), eq("request_id"));
+    expectLastCall().once();
+    dbAccessor.createIndex(eq("idx_rsc_request_id"), eq("role_success_criteria"), eq("request_id"));
+    expectLastCall().once();
+
+
+    replay(dbAccessor);
+    Module module = new Module() {
+      @Override
+      public void configure(Binder binder) {
+        binder.bind(DBAccessor.class).toInstance(dbAccessor);
+        binder.bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
+        binder.bind(EntityManager.class).toInstance(entityManager);
+      }
+    };
+
+    Injector injector = Guice.createInjector(module);
+    UpgradeCatalog221 upgradeCatalog221 = injector.getInstance(UpgradeCatalog221.class);
+    upgradeCatalog221.executeDDLUpdates();
+    verify(dbAccessor);
+  }
+
+  @Test
   public void testExecuteDMLUpdates() throws Exception {
     Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml");
     Method updateAlerts = UpgradeCatalog221.class.getDeclaredMethod("updateAlerts");


Mime
View raw message