Return-Path: X-Original-To: apmail-incubator-ambari-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-ambari-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E0E5DDA00 for ; Tue, 16 Oct 2012 00:10:43 +0000 (UTC) Received: (qmail 44946 invoked by uid 500); 16 Oct 2012 00:10:43 -0000 Delivered-To: apmail-incubator-ambari-commits-archive@incubator.apache.org Received: (qmail 44901 invoked by uid 500); 16 Oct 2012 00:10:43 -0000 Mailing-List: contact ambari-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@incubator.apache.org Delivered-To: mailing list ambari-commits@incubator.apache.org Received: (qmail 44892 invoked by uid 99); 16 Oct 2012 00:10:43 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Oct 2012 00:10:43 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Oct 2012 00:10:36 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id EBD2E23888E3; Tue, 16 Oct 2012 00:09:50 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1398583 - in /incubator/ambari/branches/AMBARI-666: ./ ambari-server/src/main/java/org/apache/ambari/server/ ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ ambari-server/src/main/java/org/apache/ambari/server/agent/ am... Date: Tue, 16 Oct 2012 00:09:50 -0000 To: ambari-commits@incubator.apache.org From: jitendra@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121016000950.EBD2E23888E3@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: jitendra Date: Tue Oct 16 00:09:49 2012 New Revision: 1398583 URL: http://svn.apache.org/viewvc?rev=1398583&view=rev Log: AMBARI-849. Stage planner implementation. Added: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/metadata/ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraphNode.java incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/stageplanner/ incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/stageplanner/TestStagePlanner.java Modified: incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/Role.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBInMemoryImpl.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionManager.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java Modified: incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt?rev=1398583&r1=1398582&r2=1398583&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt (original) +++ incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt Tue Oct 16 00:09:49 2012 @@ -12,6 +12,8 @@ AMBARI-666 branch (unreleased changes) NEW FEATURES + AMBARI-849. Stage planner implementation. (jitendra) + AMBARI-860. Remove code that adds fake stages for testing. (jitendra) AMBARI-856. Add cluster heatmap. (yusaku) Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/Role.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/Role.java?rev=1398583&r1=1398582&r2=1398583&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/Role.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/Role.java Tue Oct 16 00:09:49 2012 @@ -34,5 +34,39 @@ public enum Role { HBASE_CLIENT, JOBTRACKER, TASKTRACKER, - MAPREDUCE_CLIENT + MAPREDUCE_CLIENT, + JAVA_JCE, + KERBEROS_SERVER, + KERBEROS_CLIENT, + KERBEROS_ADMIN_CLIENT, + HADOOP_CLIENT, + JOBTRACKER_SERVICE_CHECK, + MAPREDUCE_SERVICE_CHECK, + ZOOKEEPER_SERVICE_CHECK, + ZOOKEEPER_QUORUM_SERVICE_CHECK, + HBASE_SERVICE_CHECK, + MYSQL_SERVER, + HIVE_SERVER, + HIVE_CLIENT, + HIVE_SERVICE_CHECK, + HCAT, + HCAT_SERVICE_CHECK, + OOZIE_CLIENT, + OOZIE_SERVER, + OOZIE_SERVICE_CHECK, + PIG, + PIG_SERVICE_CHECK, + SQOOP, + SQOOP_SERVICE_CHECK, + TEMPLETON_CLIENT, + TEMPLETON_SERVER, + TEMPLETON_SERVICE_CHECK, + DASHBOARD, + DASHBOARD_SERVICE_CHECK, + NAGIOS_SERVER, + GANGLIA_SERVER, + GANGLIA_MONITOR, + GMOND_SERVICE_CHECK, + GMETAD_SERVICE_CHECK, + MONTOR_WEBSERVER } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java?rev=1398583&r1=1398582&r2=1398583&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java Tue Oct 16 00:09:49 2012 @@ -54,4 +54,12 @@ public interface ActionDBAccessor { */ public long getLastPersistedRequestIdWhenInitialized(); + /** + * Updates scheduled stage. + * @param s + * @param hostname + * @param roleStr + */ + public void hostRoleScheduled(Stage s, String hostname, String roleStr); + } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java?rev=1398583&r1=1398582&r2=1398583&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java Tue Oct 16 00:09:49 2012 @@ -98,4 +98,10 @@ public class ActionDBAccessorImpl implem // TODO Auto-generated method stub return 0; } + + @Override + public void hostRoleScheduled(Stage s, String hostname, String roleStr) { + //Update start time, last update time, host role status, number of attempts + //in the database. + } } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBInMemoryImpl.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBInMemoryImpl.java?rev=1398583&r1=1398582&r2=1398583&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBInMemoryImpl.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBInMemoryImpl.java Tue Oct 16 00:09:49 2012 @@ -130,4 +130,9 @@ public class ActionDBInMemoryImpl implem public synchronized long getLastPersistedRequestIdWhenInitialized() { return lastRequestId; } + + @Override + public void hostRoleScheduled(Stage s, String hostname, String roleStr) { + //Nothing needed for in-memory implementation + } } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionManager.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionManager.java?rev=1398583&r1=1398582&r2=1398583&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionManager.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionManager.java Tue Oct 16 00:09:49 2012 @@ -57,6 +57,7 @@ public class ActionManager { } public void start() { + LOG.info("Starting scheduler thread"); scheduler.start(); } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java?rev=1398583&r1=1398582&r2=1398583&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java Tue Oct 16 00:09:49 2012 @@ -89,7 +89,7 @@ class ActionScheduler implements Runnabl private void doWork() throws AmbariException { List stages = db.getStagesInProgress(); - LOG.info("Scheduler wakes up, number of stages to look at:"+stages.size()); + LOG.info("Scheduler wakes up"); if (stages == null || stages.isEmpty()) { //Nothing to do LOG.info("No stage in progress..nothing to do"); @@ -164,7 +164,7 @@ class ActionScheduler implements Runnabl HostRoleStatus status = s.getHostRoleStatus(host, roleStr); LOG.info("Last attempt time =" + s.getLastAttemptTime(host, roleStr) + ", actiontimeout =" + this.actionTimeout + ", current time=" - + now); + + now+", status ="+status+", attempt count="+s.getAttemptCount(host, roleStr)); if (timeOutActionNeeded(status, s, host, roleStr, now)) { LOG.info("Host:" + host + ", role:" + roleStr + ", actionId:" + s.getActionId() + " timed out"); @@ -225,9 +225,7 @@ class ActionScheduler implements Runnabl long now = System.currentTimeMillis(); String roleStr = cmd.getRole().toString(); String hostname = cmd.getHostname(); - LOG.info("Host:" + hostname + ", role:" + cmd.getRole() + ", actionId:" - + s.getActionId() + " being scheduled"+", current time: "+now+", start time: "+ - s.getStartTime(hostname, roleStr)); + LOG.info("Going to schedule: "+s.toString()); if (s.getStartTime(hostname, roleStr) < 0) { LOG.info("Update state machine for first attempt"); try { @@ -259,6 +257,7 @@ class ActionScheduler implements Runnabl throw new AmbariException("Could not get string from jaxb",e); } actionQueue.enqueue(hostname, cmd); + db.hostRoleScheduled(s, hostname, roleStr); } private void updateRoleStats(HostRoleStatus status, RoleStats rs) { Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java?rev=1398583&r1=1398582&r2=1398583&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java Tue Oct 16 00:09:49 2012 @@ -121,4 +121,20 @@ class HostRoleCommand { HostRoleCommand o = (HostRoleCommand) other; return this.role.equals(o.role); } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("HostRoleCommand State:\n"); + builder.append(" Role: "+role+"\n"); + builder.append(" Status: "+status+"\n"); + builder.append(" Event: "+event+"\n"); + builder.append(" stdout: "+stdout+"\n"); + builder.append(" stderr: "+stderr+"\n"); + builder.append(" exitcode: "+exitCode+"\n"); + builder.append(" Start time: " + startTime+"\n"); + builder.append(" Last attempt time: "+lastAttemptTime+"\n"); + builder.append(" attempt count: "+ attemptCount+"\n"); + return builder.toString(); + } } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java?rev=1398583&r1=1398582&r2=1398583&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java Tue Oct 16 00:09:49 2012 @@ -135,6 +135,14 @@ public class Stage { return f; } } + + public synchronized void setSuccessFactors(Map suc) { + successFactors = suc; + } + + public synchronized Map getSuccessFactors() { + return successFactors; + } public long getRequestId() { return requestId; @@ -227,4 +235,64 @@ public class Stage { } return false; } + + public Map> getExecutionCommands() { + return this.commandsToSend; + } + + public String getLogDir() { + return this.logDir; + } + + /** + * This method should be used only in stage planner. To add + * a new execution command use + * {@link #addHostRoleExecutionCommand(String, Role, RoleCommand, + * ServiceComponentHostEvent, String, String)} + */ + public synchronized void addExecutionCommand(Stage origStage, + ExecutionCommand executionCommand) { + String hostname = executionCommand.getHostname(); + String role = executionCommand.getRole().toString(); + if (commandsToSend.get(hostname) == null) { + commandsToSend.put(hostname, new ArrayList()); + } + commandsToSend.get(hostname).add(executionCommand); + if (hostRoleCommands.get(hostname) == null) { + hostRoleCommands.put(hostname, new TreeMap()); + } + hostRoleCommands.get(hostname).put(role, + origStage.getHostRoleCommand(hostname, role)); + } + + private HostRoleCommand getHostRoleCommand(String hostname, String role) { + return hostRoleCommands.get(hostname).get(role); + } + + @Override //Object + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("STAGE DESCRIPTION BEGIN\n"); + builder.append("requestId="+requestId+"\n"); + builder.append("stageId="+stageId+"\n"); + builder.append("clusterName="+clusterName+"\n"); + builder.append("logDir=" + logDir+"\n"); + builder.append("Success Factors:\n"); + for (Role r : successFactors.keySet()) { + builder.append(" role: "+r+", factor: "+successFactors.get(r)+"\n"); + } + for (String host : commandsToSend.keySet()) { + builder.append("HOST: " + host + " :\n"); + for (ExecutionCommand ec : commandsToSend.get(host)) { + builder.append(ec.toString()); + builder.append("\n"); + HostRoleCommand hrc = hostRoleCommands.get(host).get( + ec.getRole().toString()); + builder.append(hrc.toString()); + builder.append("\n"); + } + } + builder.append("STAGE DESCRIPTION END\n"); + return builder.toString(); + } } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java?rev=1398583&r1=1398582&r2=1398583&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java Tue Oct 16 00:09:49 2012 @@ -23,6 +23,9 @@ import java.util.Map; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; +import org.apache.ambari.server.utils.StageUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.codehaus.jackson.annotate.JsonProperty; @@ -31,7 +34,9 @@ import org.codehaus.jackson.annotate.Jso * persisted in the database for recovery. */ public class ExecutionCommand extends AgentCommand { - + + private static Log LOG = LogFactory.getLog(ExecutionCommand.class); + public ExecutionCommand() { super(AgentCommandType.EXECUTION_COMMAND); } @@ -71,8 +76,12 @@ public class ExecutionCommand extends Ag @Override public String toString() { - return "Host=" + hostname + ", commandId=" + commandId + ", role=" + role - + ", command=" + roleCommand; + try { + return StageUtils.jaxbToString(this); + } catch (Exception ex) { + LOG.warn("Exception in json conversion", ex); + return "Exception in json conversion"; + } } @Override Added: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java?rev=1398583&view=auto ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java (added) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java Tue Oct 16 00:09:49 2012 @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.metadata; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.ambari.server.Role; +import org.apache.ambari.server.RoleCommand; +import org.apache.ambari.server.stageplanner.RoleGraphNode; + +/** + * This class is used to establish the order between two roles. This class + * should not be used to determine the dependencies. + */ +public class RoleCommandOrder { + + private static class RoleCommandPair { + Role role; + RoleCommand cmd; + + public RoleCommandPair(Role _role, RoleCommand _cmd) { + this.role = _role; + this.cmd = _cmd; + } + + @Override + public int hashCode() { + return (role.toString() + cmd.toString()).hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other != null && (other instanceof RoleCommandPair) + && ((RoleCommandPair) other).role.equals(role) + && ((RoleCommandPair) other).cmd.equals(cmd)) { + return true; + } + return false; + } + } + + /** + * key -> blocked role command value -> set of blocker role commands. + */ + private static Map> dependencies = new HashMap>(); + + private static void addDependency(Role blockedRole, + RoleCommand blockedCommand, Role blockerRole, RoleCommand blockerCommand) { + RoleCommandPair rcp1 = new RoleCommandPair(blockedRole, blockedCommand); + RoleCommandPair rcp2 = new RoleCommandPair(blockerRole, blockerCommand); + if (dependencies.get(rcp1) == null) { + dependencies.put(rcp1, new HashSet()); + } + dependencies.get(rcp1).add(rcp2); + } + + public static void initialize() { + addDependency(Role.SECONDARY_NAMENODE, RoleCommand.START, Role.NAMENODE, + RoleCommand.START); + addDependency(Role.HBASE_MASTER, RoleCommand.START, Role.ZOOKEEPER_SERVER, + RoleCommand.START); + addDependency(Role.HBASE_MASTER, RoleCommand.START, Role.NAMENODE, + RoleCommand.START); + addDependency(Role.HBASE_MASTER, RoleCommand.START, Role.DATANODE, + RoleCommand.START); + addDependency(Role.HBASE_REGIONSERVER, RoleCommand.START, + Role.HBASE_MASTER, RoleCommand.START); + addDependency(Role.JOBTRACKER, RoleCommand.START, Role.NAMENODE, + RoleCommand.START); + addDependency(Role.JOBTRACKER, RoleCommand.START, Role.DATANODE, + RoleCommand.START); + addDependency(Role.TASKTRACKER, RoleCommand.START, Role.JOBTRACKER, + RoleCommand.START); + addDependency(Role.OOZIE_SERVER, RoleCommand.START, Role.JOBTRACKER, + RoleCommand.START); + addDependency(Role.OOZIE_SERVER, RoleCommand.START, Role.TASKTRACKER, + RoleCommand.START); + addDependency(Role.TEMPLETON_SERVER, RoleCommand.START, Role.TASKTRACKER, + RoleCommand.START); + } + + /** + * Returns the dependency order. -1 => rgn1 before rgn2, 0 => they can be + * parallel 1 => rgn2 before rgn1 + * + * @param roleGraphNode + * @param roleGraphNode2 + */ + public int order(RoleGraphNode rgn1, RoleGraphNode rgn2) { + RoleCommandPair rcp1 = new RoleCommandPair(rgn1.getRole(), + rgn1.getCommand()); + RoleCommandPair rcp2 = new RoleCommandPair(rgn2.getRole(), + rgn2.getCommand()); + if ((dependencies.get(rcp1) != null) + && (dependencies.get(rcp1).contains(rcp2))) { + return 1; + } else if ((dependencies.get(rcp2) != null) + && (dependencies.get(rcp2).contains(rcp1))) { + return -1; + } else if (!rgn2.getCommand().equals(rgn1.getCommand())) { + return compareCommands(rgn1.getCommand(), rgn2.getCommand()); + } + return 0; + } + + private int compareCommands(RoleCommand rc1, RoleCommand rc2) { + if (rc1.equals(rc2)) { + throw new IllegalArgumentException("rc1 and rc2 are same"); + } else if (rc1.equals(RoleCommand.INSTALL)) { + return -1; + } else if (rc2.equals(RoleCommand.INSTALL)) { + return 1; + } else if (rc1.equals(RoleCommand.START)) { + return -1; + } else if (rc2.equals(RoleCommand.START)) { + return 1; + } else if (rc1.equals(RoleCommand.EXECUTE)) { + return -1; + } else if (rc2.equals(RoleCommand.EXECUTE)) { + return 1; + } else if (rc1.equals(RoleCommand.STOP)) { + return -1; + } else if (rc2.equals(RoleCommand.STOP)) { + return 1; + } + return 0; + } +} Added: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java?rev=1398583&view=auto ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java (added) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java Tue Oct 16 00:09:49 2012 @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.stageplanner; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.ambari.server.actionmanager.Stage; +import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.metadata.RoleCommandOrder; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class RoleGraph { + + private static Log LOG = LogFactory.getLog(RoleGraph.class); + + Map graph = null; + private RoleCommandOrder roleDependencies; + private Stage initialStage = null; + private boolean sameHostOptimization = true; + + public RoleGraph(RoleCommandOrder rd) { + this.roleDependencies = rd; + } + + /** + * Given a stage builds a DAG of all execution commands within the stage. + */ + public void build(Stage stage) { + if (stage == null) { + throw new IllegalArgumentException("Null stage"); + } + graph = new TreeMap(); + initialStage = stage; + Map> execCommands = stage + .getExecutionCommands(); + for (String host : execCommands.keySet()) { + for (ExecutionCommand exec : execCommands.get(host)) { + String role = exec.getRole().toString(); + RoleGraphNode rgn; + if (graph.get(role) == null) { + rgn = new RoleGraphNode(exec.getRole(), + exec.getRoleCommand()); + graph.put(role, rgn); + } + rgn = graph.get(role); + rgn.addHost(host); + } + } + //Add edges + for (String roleI : graph.keySet()) { + for (String roleJ : graph.keySet()) { + if (roleI.equals(roleJ)) { + continue; + } else { + RoleGraphNode rgnI = graph.get(roleI); + RoleGraphNode rgnJ = graph.get(roleJ); + int order = roleDependencies.order(rgnI, rgnJ); + if (order == -1) { + rgnI.addEdge(rgnJ); + } else if (order == 1) { + rgnJ.addEdge(rgnI); + } + } + } + } + } + + /** + * Returns a list of stages that need to be executed one after another + * to execute the DAG generated in the last {@link #build(Stage)} call. + */ + public List getStages() { + long initialStageId = initialStage.getStageId(); + List stageList = new ArrayList(); + List firstStageNodes = new ArrayList(); + while (!graph.isEmpty()) { + LOG.info(this.stringifyGraph()); + + for (String role: graph.keySet()) { + RoleGraphNode rgn = graph.get(role); + if (rgn.getInDegree() == 0) { + firstStageNodes.add(rgn); + } + } + Stage aStage = getStageFromGraphNodes(initialStage, firstStageNodes); + aStage.setStageId(initialStageId++); + stageList.add(aStage); + //Remove first stage nodes from the graph, we know that none of + //these nodes have an incoming edges. + for (RoleGraphNode rgn : firstStageNodes) { + if (this.sameHostOptimization) { + //Perform optimization + } + removeZeroInDegreeNode(rgn.getRole().toString()); + } + firstStageNodes.clear(); + } + return stageList; + } + + /** + * Assumes there are no incoming edges. + */ + private synchronized void removeZeroInDegreeNode(String role) { + RoleGraphNode nodeToRemove = graph.remove(role); + for (RoleGraphNode edgeNode: nodeToRemove.getEdges()) { + edgeNode.decrementInDegree(); + } + } + + private Stage getStageFromGraphNodes(Stage origStage, + List stageGraphNodes) { + Stage newStage = new Stage(origStage.getRequestId(), + origStage.getLogDir(), origStage.getClusterName()); + newStage.setSuccessFactors(origStage.getSuccessFactors()); + for (RoleGraphNode rgn : stageGraphNodes) { + for (String host : rgn.getHosts()) { + newStage.addExecutionCommand(origStage, + origStage.getExecutionCommand(host, rgn.getRole().toString())); + } + } + return newStage; + } + + public String stringifyGraph() { + StringBuilder builder = new StringBuilder(); + builder.append("Graph:\n"); + for (String role : graph.keySet()) { + builder.append(graph.get(role)); + for (RoleGraphNode rgn : graph.get(role).getEdges()) { + builder.append(" --> "); + builder.append(rgn); + } + builder.append("\n"); + } + return builder.toString(); + } +} Added: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraphNode.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraphNode.java?rev=1398583&view=auto ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraphNode.java (added) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraphNode.java Tue Oct 16 00:09:49 2012 @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.stageplanner; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.ambari.server.Role; +import org.apache.ambari.server.RoleCommand; + +public class RoleGraphNode { + public RoleGraphNode(Role role, RoleCommand command) { + this.role = role; + this.command = command; + } + private Role role; + private RoleCommand command; + private int inDegree = 0; + private List hosts = new ArrayList(); + private Map edges = new TreeMap(); + public synchronized void addHost(String host) { + hosts.add(host); + } + public synchronized void addEdge(RoleGraphNode rgn) { + if (edges.containsKey(rgn.getRole().toString())) { + return; + } + edges.put(rgn.getRole().toString(), rgn); + rgn.incrementInDegree(); + } + private synchronized void incrementInDegree() { + inDegree ++; + } + public Role getRole() { + return role; + } + public RoleCommand getCommand() { + return command; + } + public List getHosts() { + return hosts; + } + public int getInDegree() { + return inDegree; + } + + Collection getEdges() { + return edges.values(); + } + public synchronized void decrementInDegree() { + inDegree --; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("("+role+", "+command +", "+inDegree+")"); + return builder.toString(); + } +} Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java?rev=1398583&r1=1398582&r2=1398583&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java Tue Oct 16 00:09:49 2012 @@ -56,15 +56,18 @@ public class StageUtils { return requestStageIds; } + public static Stage getATestStage(long requestId, long stageId) { + String hostname; + try { + hostname = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + hostname = "host-dummy"; + } + return getATestStage(requestId, stageId, hostname); + } + //For testing only public static Stage getATestStage(long requestId, long stageId, String hostname) { - if (hostname == null || "".equals(hostname)) { - try { - hostname = InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - hostname = "host-dummy"; - } - } Stage s = new Stage(requestId, "/tmp", "cluster1"); s.setStageId(stageId); long now = System.currentTimeMillis(); @@ -90,11 +93,6 @@ public class StageUtils { Map roleParams = new TreeMap(); roleParams.put("format", "false"); execCmd.setRoleParams(roleParams); - try { - LOG.info("Command string = " + StageUtils.jaxbToString(execCmd)); - } catch (Exception e) { - throw new RuntimeException("Could not get string from jaxb",e); - } return s; } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java?rev=1398583&r1=1398582&r2=1398583&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java Tue Oct 16 00:09:49 2012 @@ -28,6 +28,8 @@ import static org.mockito.Mockito.when; import java.util.ArrayList; import java.util.List; +import junit.framework.Assert; + import org.apache.ambari.server.Role; import org.apache.ambari.server.agent.ActionQueue; import org.apache.ambari.server.agent.AgentCommand; @@ -69,20 +71,17 @@ public class TestActionScheduler { //Keep large number of attempts so that the task is not expired finally //Small action timeout to test rescheduling - ActionScheduler scheduler = new ActionScheduler(100, 100, db, aq, fsm, 10000); - // Start the thread + ActionScheduler scheduler = new ActionScheduler(100, 100, db, aq, fsm, + 10000); + // Start the thread scheduler.start(); - Thread.sleep(200); - List ac = aq.dequeueAll(hostname); - assertEquals(1, ac.size()); + List ac = waitForQueueSize(hostname, aq, 1); assertTrue(ac.get(0) instanceof ExecutionCommand); assertEquals("1-977", ((ExecutionCommand) (ac.get(0))).getCommandId()); //The action status has not changed, it should be queued again. - Thread.sleep(200); - ac = aq.dequeueAll(hostname); - assertEquals(1, ac.size()); + ac = waitForQueueSize(hostname, aq, 1); assertTrue(ac.get(0) instanceof ExecutionCommand); assertEquals("1-977", ((ExecutionCommand) (ac.get(0))).getCommandId()); @@ -91,9 +90,24 @@ public class TestActionScheduler { ac = aq.dequeueAll(hostname); //Wait for sometime, it shouldn't be scheduled this time. - Thread.sleep(200); - ac = aq.dequeueAll(hostname); - assertEquals(0, ac.size()); + ac = waitForQueueSize(hostname, aq, 0); + scheduler.stop(); + } + + private List waitForQueueSize(String hostname, ActionQueue aq, + int expectedQueueSize) throws InterruptedException { + while (true) { + List ac = aq.dequeueAll(hostname); + if (ac != null) { + if (ac.size() == expectedQueueSize) { + return ac; + } else if (ac.size() > expectedQueueSize) { + Assert.fail("Expected size : " + expectedQueueSize + " Actual size=" + + ac.size()); + } + } + Thread.sleep(100); + } } /** @@ -113,22 +127,23 @@ public class TestActionScheduler { when(scomp.getServiceComponentHost(anyString())).thenReturn(sch); when(serviceObj.getCluster()).thenReturn(oneClusterMock); - ActionDBAccessorImpl db = mock(ActionDBAccessorImpl.class); + ActionDBAccessor db = new ActionDBInMemoryImpl(); String hostname = "ahost.ambari.apache.org"; List stages = new ArrayList(); Stage s = StageUtils.getATestStage(1, 977, hostname); stages.add(s); - when(db.getStagesInProgress()).thenReturn(stages); + db.persistActions(stages); - //Keep large number of attempts so that the task is not expired finally //Small action timeout to test rescheduling - ActionScheduler scheduler = new ActionScheduler(100, 100, db, aq, fsm, 3); + ActionScheduler scheduler = new ActionScheduler(100, 50, db, aq, fsm, 3); // Start the thread scheduler.start(); - Thread.sleep(500); - //TODO timeoutHostRole must be called exactly once but in this case the state - //in the db continues to be pending therefore it is processed multiple times. - verify(db, atLeastOnce()).timeoutHostRole(hostname, 1, 977, Role.NAMENODE); + while (!stages.get(0).getHostRoleStatus(hostname, "NAMENODE") + .equals(HostRoleStatus.TIMEDOUT)) { + Thread.sleep(100); + } + assertEquals(stages.get(0).getHostRoleStatus(hostname, "NAMENODE"), + HostRoleStatus.TIMEDOUT); } } Added: incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/stageplanner/TestStagePlanner.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/stageplanner/TestStagePlanner.java?rev=1398583&view=auto ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/stageplanner/TestStagePlanner.java (added) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/stageplanner/TestStagePlanner.java Tue Oct 16 00:09:49 2012 @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.stageplanner; + +import static org.junit.Assert.*; + +import java.util.List; + +import org.apache.ambari.server.Role; +import org.apache.ambari.server.RoleCommand; +import org.apache.ambari.server.actionmanager.Stage; +import org.apache.ambari.server.metadata.RoleCommandOrder; +import org.apache.ambari.server.state.svccomphost.ServiceComponentHostInstallEvent; +import org.apache.ambari.server.state.svccomphost.ServiceComponentHostStartEvent; +import org.apache.ambari.server.utils.StageUtils; +import org.junit.Test; + +public class TestStagePlanner { + + @Test + public void testSingleStagePlan() { + RoleCommandOrder.initialize(); + RoleCommandOrder rco = new RoleCommandOrder(); + RoleGraph rg = new RoleGraph(rco); + String hostname = "dummy"; + Stage stage = StageUtils.getATestStage(1, 1, hostname); + rg.build(stage); + List outStages = rg.getStages(); + for (Stage s: outStages) { + System.out.println(s.toString()); + } + assertEquals(1, outStages.size()); + assertEquals(stage.getExecutionCommands(hostname), outStages.get(0) + .getExecutionCommands(hostname)); + } + + @Test + public void testMultiStagePlan() { + RoleCommandOrder.initialize(); + RoleCommandOrder rco = new RoleCommandOrder(); + RoleGraph rg = new RoleGraph(rco); + long now = System.currentTimeMillis(); + Stage stage = StageUtils.getATestStage(1, 1, "host1"); + stage.addHostRoleExecutionCommand("host2", Role.HBASE_MASTER, + RoleCommand.START, new ServiceComponentHostStartEvent("HBASE_MASTER", + "host2", now), "cluster1", "HBASE"); + stage.addHostRoleExecutionCommand("host3", Role.ZOOKEEPER_SERVER, + RoleCommand.START, new ServiceComponentHostStartEvent("ZOOKEEPER_SERVER", + "host3", now), "cluster1", "ZOOKEEPER"); + System.out.println(stage.toString()); + + rg.build(stage); + System.out.println(rg.stringifyGraph()); + List outStages = rg.getStages(); + for (Stage s: outStages) { + System.out.println(s.toString()); + } + assertEquals(3, outStages.size()); + } + + @Test + public void testManyStages() { + RoleCommandOrder.initialize(); + RoleCommandOrder rco = new RoleCommandOrder(); + RoleGraph rg = new RoleGraph(rco); + long now = System.currentTimeMillis(); + Stage stage = StageUtils.getATestStage(1, 1, "host1"); + stage.addHostRoleExecutionCommand("host11", Role.SECONDARY_NAMENODE, + RoleCommand.START, new ServiceComponentHostStartEvent("SECONDARY_NAMENODE", + "host11", now), "cluster1", "HDFS"); + stage.addHostRoleExecutionCommand("host2", Role.HBASE_MASTER, + RoleCommand.START, new ServiceComponentHostStartEvent("HBASE_MASTER", + "host2", now), "cluster1", "HBASE"); + stage.addHostRoleExecutionCommand("host3", Role.ZOOKEEPER_SERVER, + RoleCommand.START, new ServiceComponentHostStartEvent("ZOOKEEPER_SERVER", + "host3", now), "cluster1", "ZOOKEEPER"); + stage.addHostRoleExecutionCommand("host4", Role.DATANODE, + RoleCommand.START, new ServiceComponentHostStartEvent("DATANODE", + "host4", now), "cluster1", "HDFS"); + stage.addHostRoleExecutionCommand("host4", Role.HBASE_REGIONSERVER, + RoleCommand.START, new ServiceComponentHostStartEvent("HBASE_REGIONSERVER", + "host4", now), "cluster1", "HBASE"); + stage.addHostRoleExecutionCommand("host4", Role.TASKTRACKER, + RoleCommand.START, new ServiceComponentHostStartEvent("TASKTRACKER", + "host4", now), "cluster1", "MAPREDUCE"); + stage.addHostRoleExecutionCommand("host5", Role.JOBTRACKER, + RoleCommand.START, new ServiceComponentHostStartEvent("JOBTRACKER", + "host5", now), "cluster1", "MAPREDUCE"); + stage.addHostRoleExecutionCommand("host6", Role.OOZIE_SERVER, + RoleCommand.START, new ServiceComponentHostStartEvent("OOZIE_SERVER", + "host6", now), "cluster1", "OOZIE"); + stage.addHostRoleExecutionCommand("host7", Role.TEMPLETON_SERVER, + RoleCommand.START, new ServiceComponentHostStartEvent("TEMPLETON_SERVER", + "host7", now), "cluster1", "TEMPLETON"); + stage.addHostRoleExecutionCommand("host8", Role.NAGIOS_SERVER, + RoleCommand.START, new ServiceComponentHostStartEvent("NAGIOS_SERVER", + "host8", now), "cluster1", "NAGIOS"); + stage.addHostRoleExecutionCommand("host4", Role.GANGLIA_MONITOR, + RoleCommand.START, new ServiceComponentHostStartEvent("GANGLIA_MONITOR", + "host4", now), "cluster1", "GANGLIA"); + stage.addHostRoleExecutionCommand("host9", Role.GANGLIA_SERVER, + RoleCommand.START, new ServiceComponentHostStartEvent("GANGLIA_SERVER", + "host9", now), "cluster1", "GANGLIA"); + System.out.println(stage.toString()); + rg.build(stage); + System.out.println(rg.stringifyGraph()); + List outStages = rg.getStages(); + for (Stage s: outStages) { + System.out.println(s.toString()); + } + assertEquals(5, outStages.size()); + } +}