helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zzh...@apache.org
Subject git commit: [HELIX-315] Handle alerts in controller, rb=17758
Date Thu, 13 Feb 2014 22:36:29 GMT
Updated Branches:
  refs/heads/helix-monitoring a99284237 -> b182690f8


[HELIX-315] Handle alerts in controller, rb=17758


Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/b182690f
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/b182690f
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/b182690f

Branch: refs/heads/helix-monitoring
Commit: b182690f866b5c4617b60b6790ded8abcfba46b3
Parents: a992842
Author: zzhang <zzhang5@uci.edu>
Authored: Thu Feb 13 14:36:17 2014 -0800
Committer: zzhang <zzhang5@uci.edu>
Committed: Thu Feb 13 14:36:17 2014 -0800

----------------------------------------------------------------------
 .../main/java/org/apache/helix/HelixAdmin.java  |   9 +
 .../main/java/org/apache/helix/PropertyKey.java |  11 ++
 .../helix/controller/alert/AlertAction.java     | 142 +++++++++++++++
 .../helix/controller/alert/AlertComparator.java |  54 ++++++
 .../helix/controller/alert/AlertName.java       | 151 +++++++++++++++
 .../helix/controller/alert/AlertScope.java      | 106 +++++++++++
 .../zk/DefaultAlertMsgHandlerFactory.java       | 122 +++++++++++++
 .../apache/helix/manager/zk/ZKHelixAdmin.java   | 102 ++++++++++-
 .../manager/zk/ZkCacheBaseDataAccessor.java     |   5 +-
 .../org/apache/helix/model/AlertConfig.java     |  89 +++++++++
 .../apache/helix/model/HelixConfigScope.java    |  21 ++-
 .../java/org/apache/helix/model/Message.java    |   6 +-
 .../org/apache/helix/tools/ClusterSetup.java    | 165 +++++------------
 .../org/apache/helix/util/StatusUpdateUtil.java |   2 +-
 .../helix/controller/alert/TestAlertAction.java |  95 ++++++++++
 .../helix/controller/alert/TestAlertName.java   | 110 +++++++++++
 .../helix/controller/alert/TestAlertScope.java  |  78 ++++++++
 .../manager/zk/TestDefaultAlertMsgHandler.java  | 182 +++++++++++++++++++
 .../helix/manager/zk/TestZkHelixAdmin.java      |   1 +
 .../org/apache/helix/model/TestAlertConfig.java |  71 ++++++++
 20 files changed, 1389 insertions(+), 133 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
index 892bada..681b938 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
@@ -424,4 +424,13 @@ public interface HelixAdmin {
    * Release resources
    */
   void close();
+
+  /**
+   * Swap instances
+   * @param clusterName
+   * @param oldInstanceName
+   * @param newInstanceName
+   */
+  void swapInstance(String clusterName, String oldInstanceName, String newInstanceName);
+
 }

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/main/java/org/apache/helix/PropertyKey.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyKey.java b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
index 4ee3f83..e1c8f5f 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyKey.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
@@ -46,6 +46,7 @@ import static org.apache.helix.PropertyType.STATUSUPDATES_CONTROLLER;
 
 import java.util.Arrays;
 
+import org.apache.helix.model.AlertConfig;
 import org.apache.helix.model.AlertHistory;
 import org.apache.helix.model.AlertStatus;
 import org.apache.helix.model.Alerts;
@@ -344,6 +345,16 @@ public class PropertyKey {
     }
 
     /**
+    * Get a property key associated with a single {@link AlertConfig}
+    * @param alertConfigName name of the configuration
+    * @return {@link PropertyKey}
+    */
+    public PropertyKey alertConfig(String alertConfigName) {
+      return new PropertyKey(CONFIGS, ConfigScopeProperty.ALERT, AlertConfig.class,
+           _clusterName, ConfigScopeProperty.ALERT.name(), alertConfigName);
+    }
+
+    /**
      * Get a property key associated with {@link LiveInstance}
      * @return {@link PropertyKey}
      */

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/main/java/org/apache/helix/controller/alert/AlertAction.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/alert/AlertAction.java b/helix-core/src/main/java/org/apache/helix/controller/alert/AlertAction.java
new file mode 100644
index 0000000..90efe2b
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/controller/alert/AlertAction.java
@@ -0,0 +1,142 @@
+package org.apache.helix.controller.alert;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Arrays;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.Options;
+import org.apache.helix.tools.ClusterSetup;
+
+public class AlertAction {
+  private final String[] _cliArgs;
+  private final CommandLine _cli;
+
+  public static class Builder {
+    private String _cmd = null;
+    private String[] _args = null;
+
+    public Builder cmd(String cmd) {
+      _cmd = cmd;
+      return this;
+    }
+
+    public Builder args(String... args) {
+      _args = args;
+      return this;
+    }
+
+    public AlertAction build() {
+      return new AlertAction(_cmd, _args);
+    }
+  }
+
+  public AlertAction(String cmd, String[] args) {
+    if (cmd == null) {
+      throw new NullPointerException("command is null");
+    }
+    if (args == null) {
+      args = new String[0];
+    }
+
+    // if arg is in form of "{..}", make sure it's a valid
+    // alertScope
+    for (String arg : args) {
+      if (arg.startsWith("{") && arg.endsWith("}")) {
+        try {
+          String filedStr = arg.substring(1, arg.length() - 1);
+          AlertScope.AlertScopeField.valueOf(filedStr);
+        } catch (IllegalArgumentException e) {
+          throw new IllegalArgumentException("Invalid alertScope value: " + arg + " in "
+              + Arrays.asList(args));
+        }
+      }
+    }
+
+    _cliArgs = new String[args.length + 1];
+    _cliArgs[0] = cmd;
+    System.arraycopy(args, 0, _cliArgs, 1, args.length);
+    _cli = parseCliArgs();
+
+  }
+
+  /**
+   * represent alertAction in form of: (command)(arg1 arg2 ...)
+   */
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("(").append(_cliArgs[0]).append(")");
+    if (_cliArgs != null) {
+      sb.append("(");
+      for (int i = 1; i < _cliArgs.length; i++) {
+        String arg = _cliArgs[i];
+        if (i > 1) {
+          sb.append(" ");
+        }
+        sb.append(arg);
+      }
+      sb.append(")");
+    }
+    return sb.toString();
+  }
+
+  public static AlertAction from(String actionStr) {
+    if (actionStr == null) {
+      throw new NullPointerException("actionStr is null");
+    }
+
+    // split (command)(arg1 arg2 ..) to [ , command, , arg1 arg2 ...]
+    String[] parts = actionStr.split("[()]");
+
+    if (parts.length == 2 && parts[0].isEmpty()) {
+      return new AlertAction(parts[1], null);
+    } else if (parts.length == 4 && parts[0].isEmpty() && parts[2].isEmpty()) {
+      String[] args = parts[3].split("\\s+");
+      return new AlertAction(parts[1], args);
+    }
+    throw new IllegalArgumentException("Invalid alerAction string, was " + actionStr);
+  }
+
+  public String[] getCliArgs() {
+    return _cliArgs;
+  }
+
+  public CommandLine getCli() {
+    return _cli;
+  }
+
+  private CommandLine parseCliArgs() {
+    Options options = new Options();
+    options.addOptionGroup(ClusterSetup.constructOptionGroup());
+
+    try {
+      String[] tmpCliArgs = Arrays.copyOf(_cliArgs, _cliArgs.length);
+
+      // prepend "-" to helix-command (_cliArgs[0]), so it can be parsed
+      tmpCliArgs[0] = String.format("-%s", tmpCliArgs[0]);
+      return new GnuParser().parse(options, tmpCliArgs);
+    } catch (Exception e) {
+      throw new IllegalArgumentException(
+          "Invalid HelixAdmin command: " + Arrays.toString(_cliArgs), e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/main/java/org/apache/helix/controller/alert/AlertComparator.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/alert/AlertComparator.java b/helix-core/src/main/java/org/apache/helix/controller/alert/AlertComparator.java
new file mode 100644
index 0000000..6bb1426
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/controller/alert/AlertComparator.java
@@ -0,0 +1,54 @@
+package org.apache.helix.controller.alert;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public enum AlertComparator {
+  LARGER(">"),
+  SMALLER("<"),
+  NOT_EQUAL("!=");
+
+  private static final Map<String, AlertComparator> _indexMap;
+  private final String _op;
+
+  static {
+    Map<String, AlertComparator> aMap = new HashMap<String, AlertComparator>();
+    for (AlertComparator cmp : AlertComparator.values()) {
+      aMap.put(cmp._op, cmp);
+    }
+    _indexMap = Collections.unmodifiableMap(aMap);
+  }
+
+  private AlertComparator(String op) {
+    _op = op;
+  }
+
+  @Override
+  public String toString() {
+    return _op;
+  }
+
+  public static AlertComparator from(String op) {
+    return _indexMap.get(op);
+  }
+}

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/main/java/org/apache/helix/controller/alert/AlertName.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/alert/AlertName.java b/helix-core/src/main/java/org/apache/helix/controller/alert/AlertName.java
new file mode 100644
index 0000000..0fdd3a1
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/controller/alert/AlertName.java
@@ -0,0 +1,151 @@
+package org.apache.helix.controller.alert;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.helix.api.id.ClusterId;
+import org.apache.helix.api.id.Id;
+import org.apache.helix.api.id.PartitionId;
+import org.apache.helix.api.id.ResourceId;
+
+public class AlertName {
+  final AlertScope _scope;
+  final String _metric;
+
+  final AlertComparator _comparator;
+  final String _value;
+
+  public static class Builder {
+    private ClusterId _clusterId = null;
+    private Id _tenantId = null;
+    private Id _nodeId = null;
+    private ResourceId _resourceId = null;
+    private PartitionId _partitionId = null;
+    private String _metric = null;
+    private AlertComparator _comparator = null;
+    private String _value = null;
+
+    public Builder cluster(ClusterId clusterId) {
+      _clusterId = clusterId;
+      return this;
+    }
+
+    public Builder tenant(Id tenantId) {
+      _tenantId = tenantId;
+      return this;
+    }
+
+    public Builder node(Id nodeId) {
+      _nodeId = nodeId;
+      return this;
+    }
+
+    public Builder resource(ResourceId resourceId) {
+      _resourceId = resourceId;
+      return this;
+    }
+
+    public Builder partitionId(PartitionId partitionId) {
+      _partitionId = partitionId;
+      return this;
+    }
+
+    public Builder metric(String metric) {
+      _metric = metric;
+      return this;
+    }
+
+    public Builder largerThan(String value) {
+      _comparator = AlertComparator.LARGER;
+      _value = value;
+      return this;
+    }
+
+    public Builder smallerThan(String value) {
+      _comparator = AlertComparator.SMALLER;
+      _value = value;
+      return this;
+    }
+
+    public AlertName build() {
+      return new AlertName(
+          new AlertScope(_clusterId, _tenantId, _nodeId, _resourceId, _partitionId), _metric,
+          _comparator, _value);
+    }
+  }
+
+  public AlertName(AlertScope scope, String metric, AlertComparator comparator, String value) {
+    if (scope == null || metric == null || comparator == null || value == null) {
+      throw new NullPointerException("null arguments, was scope: " + scope + ", metric: " + metric
+          + ", comparator: " + comparator + ", value: " + value);
+    }
+    _scope = scope;
+    _metric = metric;
+    _comparator = comparator;
+    _value = value;
+  }
+
+  /**
+   * represent alertName in form of:
+   * (scope)(metric)comparator(value)
+   */
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("(").append(_scope.toString()).append(")");
+    sb.append("(").append(_metric).append(")");
+    sb.append(_comparator.toString());
+    sb.append("(").append(_value).append(")");
+    return sb.toString();
+  }
+
+  public static AlertName from(String alertNameStr) {
+    if (alertNameStr == null) {
+      throw new NullPointerException("alertNameStr is null");
+    }
+
+    // split (alertName)(metric)cmp(value) to [ , alertName, , metric, cmp, value]
+    String[] parts = alertNameStr.split("[()]");
+    if (parts == null || parts.length != 6 || !parts[0].isEmpty() || !parts[2].isEmpty()) {
+      throw new IllegalArgumentException(
+          "AlertName is NOT in form of (scope)(metric)comparator(value), was " + alertNameStr);
+    }
+
+    String[] scopeParts = parts[1].split("\\.");
+
+    String metric = parts[3];
+    AlertComparator cmp = AlertComparator.from(parts[4]);
+    if (cmp == null) {
+      throw new IllegalArgumentException("Invalid alert comparator, was " + parts[4]);
+    }
+    String value = parts[5];
+
+    AlertScope scope = new AlertScope(scopeParts);
+    return new AlertName(scope, metric, cmp, value);
+  }
+
+  public boolean match(AlertName alertName) {
+    return _scope.match(alertName._scope) && _metric.equals(alertName._metric)
+        && _comparator == alertName._comparator && _value.equals(alertName._value);
+  }
+
+  public AlertScope getScope() {
+    return _scope;
+  }
+}

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/main/java/org/apache/helix/controller/alert/AlertScope.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/alert/AlertScope.java b/helix-core/src/main/java/org/apache/helix/controller/alert/AlertScope.java
new file mode 100644
index 0000000..bc32ce6
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/controller/alert/AlertScope.java
@@ -0,0 +1,106 @@
+package org.apache.helix.controller.alert;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Arrays;
+
+import org.apache.helix.api.id.ClusterId;
+import org.apache.helix.api.id.Id;
+
+public class AlertScope {
+  public static final String WILDCARD = "%";
+
+  /**
+   * DON'T change the sequence
+   */
+  public enum AlertScopeField {
+    cluster,
+    tenant,
+    node,
+    resource,
+    partition;
+  }
+
+  private final String[] _scopeFields;
+
+  public AlertScope(Id clusterId, Id tenantId, Id nodeId, Id resourceId, Id partitionId) {
+    this(clusterId == null ? null : clusterId.stringify(), tenantId == null ? null : tenantId
+        .stringify(), nodeId == null ? null : nodeId.stringify(), resourceId == null ? null
+        : resourceId.stringify(), partitionId == null ? null : partitionId.stringify());
+  }
+
+  public AlertScope(String... fields) {
+    int maxArgNum = AlertScopeField.values().length;
+    if (fields != null && fields.length > maxArgNum) {
+      throw new IllegalArgumentException("Too many arguments. Should be no more than " + maxArgNum
+           + " but was " + fields.length + ", fields: " + Arrays.asList(fields));
+    }
+    // all array elements are init'ed to null
+    _scopeFields = new String[AlertScopeField.values().length];
+
+    if (fields != null) {
+      for (int i = 0; i < fields.length; i++) {
+        if (!WILDCARD.equals(fields[i])) {
+          _scopeFields[i] = fields[i];
+        }
+      }
+    }
+  }
+
+  /**
+   * represent AlertScope in form of:
+   * {cluster}.{tenant}.{node}.{resource}.{partition} using % for null field
+   */
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    for (String scopeField : _scopeFields) {
+      if (sb.length() > 0) {
+        sb.append(".");
+      }
+      sb.append(scopeField == null ? WILDCARD : scopeField);
+    }
+
+    return sb.toString();
+  }
+
+  public String get(AlertScopeField field) {
+    return _scopeFields[field.ordinal()];
+  }
+
+  public ClusterId getClusterId() {
+    return ClusterId.from(get(AlertScopeField.cluster));
+  }
+
+  /**
+   * match two alert-scopes, null means "don't care"
+   * @param scope
+   * @return
+   */
+  public boolean match(AlertScope scope) {
+    for (int i = 0; i < _scopeFields.length; i++) {
+      if (_scopeFields[i] != null && scope._scopeFields[i] != null
+          && !_scopeFields[i].equals(scope._scopeFields[i])) {
+        return false;
+      }
+    }
+    return true;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/main/java/org/apache/helix/manager/zk/DefaultAlertMsgHandlerFactory.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/DefaultAlertMsgHandlerFactory.java b/helix-core/src/main/java/org/apache/helix/manager/zk/DefaultAlertMsgHandlerFactory.java
new file mode 100644
index 0000000..814f01c
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/DefaultAlertMsgHandlerFactory.java
@@ -0,0 +1,122 @@
+package org.apache.helix.manager.zk;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixException;
+import org.apache.helix.HelixManager;
+import org.apache.helix.NotificationContext;
+import org.apache.helix.messaging.handling.HelixTaskResult;
+import org.apache.helix.messaging.handling.MessageHandler;
+import org.apache.helix.messaging.handling.MessageHandlerFactory;
+import org.apache.helix.controller.alert.AlertAction;
+import org.apache.helix.model.AlertConfig;
+import org.apache.helix.controller.alert.AlertName;
+import org.apache.helix.model.Message;
+import org.apache.helix.model.Message.Attributes;
+import org.apache.helix.tools.ClusterSetup;
+import org.apache.log4j.Logger;
+
+public class DefaultAlertMsgHandlerFactory implements MessageHandlerFactory {
+  private static final Logger LOG = Logger.getLogger(DefaultAlertMsgHandlerFactory.class);
+  public static final String DEFAULT_ALERT_CONFIG = "default";
+
+  public static class DefaultAlertMsgHandler extends MessageHandler {
+    public DefaultAlertMsgHandler(Message message, NotificationContext context) {
+      super(message, context);
+    }
+
+    @Override
+    public HelixTaskResult handleMessage() throws InterruptedException {
+      HelixManager manager = _notificationContext.getManager();
+      HelixTaskResult result = new HelixTaskResult();
+
+      // get alert-name from message
+      String alertNameStr = _message.getAttribute(Attributes.ALERT_NAME);
+      AlertName alertName = AlertName.from(alertNameStr);
+
+      // get action from alert config
+      HelixDataAccessor accessor = manager.getHelixDataAccessor();
+      AlertConfig defaultAlertConfig =
+          accessor.getProperty(accessor.keyBuilder().alertConfig(DEFAULT_ALERT_CONFIG));
+      AlertAction action = defaultAlertConfig.findAlertAction(alertName);
+
+      if (action != null) {
+        // perform action
+        HelixAdmin admin = manager.getClusterManagmentTool();
+        try {
+          ClusterSetup setupTool = new ClusterSetup(admin);
+          ClusterSetup.processCommandLineArgs(setupTool, action.getCli());
+          result.setSuccess(true);
+
+        } catch (Exception e) {
+          String errMsg = "Exception execute action: " + action + " for alert: " + alertNameStr;
+          result.setSuccess(false);
+          result.setMessage(errMsg);
+          result.setException(e);
+          LOG.error(errMsg, e);
+        }
+      } else {
+        String errMsg = "Could NOT find action for alert: " + alertNameStr;
+        result.setSuccess(false);
+        result.setMessage(errMsg);
+        LOG.error(errMsg);
+      }
+
+      return result;
+    }
+
+    @Override
+    public void onError(Exception e, ErrorCode code, ErrorType type) {
+      LOG.error("Error processing message: " + _message + ", errCode: " + code + ", errType: "
+          + type, e);
+    }
+
+  }
+
+  public DefaultAlertMsgHandlerFactory() {
+    LOG.info("construct default alert message handler factory");
+  }
+
+  @Override
+  public MessageHandler createHandler(Message message, NotificationContext context) {
+    String type = message.getMsgType();
+
+    if (!type.equals(getMessageType())) {
+      throw new HelixException("Unexpected msg type for message " + message.getMessageId()
+           + " type:" + message.getMsgType());
+    }
+
+    return new DefaultAlertMsgHandler(message, context);
+
+  }
+
+  @Override
+  public String getMessageType() {
+    return Message.MessageType.ALERT.name();
+  }
+
+  @Override
+  public void reset() {
+    LOG.info("reset default alert message handler factory");
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
index 6e30074..5aae1f7 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
@@ -86,6 +86,8 @@ import org.apache.helix.util.RebalanceUtil;
 import org.apache.log4j.Logger;
 
 public class ZKHelixAdmin implements HelixAdmin {
+  static Logger LOG = Logger.getLogger(ZKHelixAdmin.class);
+
   public static final String CONNECTION_TIMEOUT = "helixAdmin.timeOutInSec";
   private final ZkClient _zkClient;
   private final ConfigAccessor _configAccessor;
@@ -130,7 +132,10 @@ public class ZKHelixAdmin implements HelixAdmin {
 
   @Override
   public void dropInstance(String clusterName, InstanceConfig instanceConfig) {
-    // String instanceConfigsPath = HelixUtil.getConfigPath(clusterName);
+    ZKHelixDataAccessor accessor =
+        new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
+    Builder keyBuilder = accessor.keyBuilder();
+
     String instanceConfigsPath =
         PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName,
             ConfigScopeProperty.PARTICIPANT.toString());
@@ -148,6 +153,29 @@ public class ZKHelixAdmin implements HelixAdmin {
           + clusterName);
     }
 
+    String instanceId = instanceConfig.getInstanceName();
+
+    // ensure node is stopped
+    LiveInstance liveInstance = accessor.getProperty(keyBuilder.liveInstance(instanceId));
+    if (liveInstance != null) {
+      throw new HelixException("Can't drop " + instanceId + ", please stop " + instanceId
+          + " before drop it");
+    }
+
+    InstanceConfig config = accessor.getProperty(keyBuilder.instanceConfig(instanceId));
+    if (config == null) {
+      String error = "Node " + instanceId + " does not exist, cannot drop";
+      LOG.warn(error);
+      throw new HelixException(error);
+    }
+
+    // ensure node is disabled, otherwise fail
+    if (config.getInstanceEnabled()) {
+      String error = "Node " + instanceId + " is enabled, cannot drop";
+      LOG.warn(error);
+      throw new HelixException(error);
+    }
+
     // delete config path
     ZKUtil.dropChildren(_zkClient, instanceConfigsPath, instanceConfig.getRecord());
 
@@ -917,7 +945,8 @@ public class ZKHelixAdmin implements HelixAdmin {
         new ZKHelixDataAccessor(grandCluster, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
     Builder keyBuilder = accessor.keyBuilder();
 
-    accessor.setProperty(keyBuilder.idealStates(idealState.getResourceId().stringify()), idealState);
+    accessor
+        .setProperty(keyBuilder.idealStates(idealState.getResourceId().stringify()), idealState);
   }
 
   @Override
@@ -1237,10 +1266,79 @@ public class ZKHelixAdmin implements HelixAdmin {
     accessor.setProperty(keyBuilder.instanceConfig(instanceName), config);
   }
 
+  @Override
   public void close() {
     if (_zkClient != null) {
       _zkClient.close();
     }
   }
 
+  @Override
+  public void swapInstance(String clusterName, String oldInstanceName, String newInstanceName) {
+    ZKHelixDataAccessor accessor =
+        new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
+    Builder keyBuilder = accessor.keyBuilder();
+
+    InstanceConfig oldConfig = accessor.getProperty(keyBuilder.instanceConfig(oldInstanceName));
+    if (oldConfig == null) {
+      String error = "Old instance " + oldInstanceName + " does not exist, cannot swap";
+      LOG.warn(error);
+      throw new HelixException(error);
+    }
+
+    InstanceConfig newConfig = accessor.getProperty(keyBuilder.instanceConfig(newInstanceName));
+    if (newConfig == null) {
+      String error = "New instance " + newInstanceName + " does not exist, cannot swap";
+      LOG.warn(error);
+      throw new HelixException(error);
+    }
+
+    // ensure old instance is disabled, otherwise fail
+    if (oldConfig.getInstanceEnabled()) {
+      String error =
+          "Old instance " + oldInstanceName + " is enabled, it need to be disabled and turned off";
+      LOG.warn(error);
+      throw new HelixException(error);
+    }
+    // ensure old instance is down, otherwise fail
+    List<String> liveInstanceNames = accessor.getChildNames(accessor.keyBuilder().liveInstances());
+
+    if (liveInstanceNames.contains(oldInstanceName)) {
+      String error =
+          "Old instance " + oldInstanceName + " is still on, it need to be disabled and turned off";
+      LOG.warn(error);
+      throw new HelixException(error);
+    }
+
+    dropInstance(clusterName, oldConfig);
+
+    List<IdealState> existingIdealStates =
+        accessor.getChildValues(accessor.keyBuilder().idealStates());
+    for (IdealState idealState : existingIdealStates) {
+      swapInstanceInIdealState(idealState, oldInstanceName, newInstanceName);
+      accessor.setProperty(accessor.keyBuilder()
+          .idealStates(idealState.getResourceId().stringify()), idealState);
+    }
+  }
+
+  private void swapInstanceInIdealState(IdealState idealState, String oldInstance,
+      String newInstance) {
+    for (String partition : idealState.getRecord().getMapFields().keySet()) {
+      Map<String, String> valMap = idealState.getRecord().getMapField(partition);
+      if (valMap.containsKey(oldInstance)) {
+        valMap.put(newInstance, valMap.get(oldInstance));
+        valMap.remove(oldInstance);
+      }
+    }
+
+    for (String partition : idealState.getRecord().getListFields().keySet()) {
+      List<String> valList = idealState.getRecord().getListField(partition);
+      for (int i = 0; i < valList.size(); i++) {
+        if (valList.get(i).equals(oldInstance)) {
+          valList.remove(i);
+          valList.add(i, newInstance);
+        }
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/main/java/org/apache/helix/manager/zk/ZkCacheBaseDataAccessor.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZkCacheBaseDataAccessor.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZkCacheBaseDataAccessor.java
index a36c49a..0d0bbe4 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZkCacheBaseDataAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZkCacheBaseDataAccessor.java
@@ -505,11 +505,10 @@ public class ZkCacheBaseDataAccessor<T> implements HelixPropertyStore<T> {
   @Override
   public boolean[] exists(List<String> paths, int options) {
     final int size = paths.size();
-    List<String> serverPaths = prependChroot(paths);
 
     boolean exists[] = new boolean[size];
     for (int i = 0; i < size; i++) {
-      exists[i] = exists(serverPaths.get(i), options);
+      exists[i] = exists(paths.get(i), options);
     }
     return exists;
   }
@@ -655,7 +654,7 @@ public class ZkCacheBaseDataAccessor<T> implements HelixPropertyStore<T> {
 
     List<String> paths = new ArrayList<String>();
     for (String childName : childNames) {
-      String path = parentPath + "/" + childName;
+      String path = parentPath.equals("/")? "/" + childName : parentPath + "/" + childName;
       paths.add(path);
     }
 

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/main/java/org/apache/helix/model/AlertConfig.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/model/AlertConfig.java b/helix-core/src/main/java/org/apache/helix/model/AlertConfig.java
new file mode 100644
index 0000000..54da31d
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/model/AlertConfig.java
@@ -0,0 +1,89 @@
+package org.apache.helix.model;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.helix.HelixProperty;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.controller.alert.AlertAction;
+import org.apache.helix.controller.alert.AlertName;
+import org.apache.helix.controller.alert.AlertScope;
+import org.apache.log4j.Logger;
+
+public class AlertConfig extends HelixProperty {
+  private static final Logger LOG = Logger.getLogger(AlertConfig.class);
+
+  public AlertConfig(ZNRecord record) {
+    super(record);
+  }
+
+  public AlertConfig(String id) {
+    super(id);
+  }
+
+  public void putConfig(AlertName alertName, AlertAction action) {
+    _record.setSimpleField(alertName.toString(), action.toString());
+  }
+
+  /**
+   * Given an alert name, find the matching action
+   * @param matchingName
+   * @return
+   */
+  public AlertAction findAlertAction(AlertName matchingName) {
+    // search each alert config entry to find a match
+    for (String alertNameStr : _record.getSimpleFields().keySet()) {
+      String actionStr = _record.getSimpleField(alertNameStr);
+
+      try {
+        AlertName name = AlertName.from(alertNameStr);
+        if (!name.match(matchingName)) {
+          continue;
+        }
+
+        // find a match
+        // replace "{scope}" in action.args with actual values
+        // e.g. "{node}" -> "localhost_12918"
+        int start = actionStr.indexOf("{");
+        while (start != -1) {
+          int end = actionStr.indexOf("}", start + 1);
+          String fieldStr = actionStr.substring(start + 1, end);
+          AlertScope.AlertScopeField field = AlertScope.AlertScopeField.valueOf(fieldStr);
+
+          String fieldValue = matchingName.getScope().get(field);
+          if (fieldValue == null) {
+            throw new NullPointerException("Null value for alert scope field: " + field
+                + ", in alert: " + matchingName);
+          }
+
+          actionStr = actionStr.replace("{" + fieldStr + "}", fieldValue);
+          start = actionStr.indexOf("{");
+        }
+
+        return AlertAction.from(actionStr);
+      } catch (Exception e) {
+        LOG.error("Exception in find action for alert: " + matchingName
+            + ", matching alertEntry: \"" + alertNameStr + ":" + actionStr + "\"", e);
+      }
+    }
+
+    return null;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java b/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
index 40d5671..3492187 100644
--- a/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
+++ b/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
@@ -36,7 +36,8 @@ public class HelixConfigScope {
     RESOURCE(2, 0),
     PARTITION(2, 1),
     CONSTRAINT(2, 0),
-    MONITORING(2, 0);
+    MONITORING(2, 0),
+    ALERT(2, 0);
 
     final int _zkPathArgNum;
     final int _mapKeyArgNum;
@@ -80,12 +81,14 @@ public class HelixConfigScope {
         "/{clusterName}/CONFIGS/RESOURCE/{resourceName}");
     template.addEntry(ConfigScopeProperty.MONITORING, 2,
         "/{clusterName}/CONFIGS/MONITORING/{configName}");
+    template.addEntry(ConfigScopeProperty.ALERT, 2, "/{clusterName}/CONFIGS/ALERT/{configName}");
 
     // get children
     template.addEntry(ConfigScopeProperty.CLUSTER, 1, "/{clusterName}/CONFIGS/CLUSTER");
     template.addEntry(ConfigScopeProperty.PARTICIPANT, 1, "/{clusterName}/CONFIGS/PARTICIPANT");
     template.addEntry(ConfigScopeProperty.RESOURCE, 1, "/{clusterName}/CONFIGS/RESOURCE");
     template.addEntry(ConfigScopeProperty.MONITORING, 1, "/{clusterName}/CONFIGS/MONITORING");
+    template.addEntry(ConfigScopeProperty.ALERT, 1, "/{clusterName}/CONFIGS/ALERT");
   }
 
   final ConfigScopeProperty _type;
@@ -100,6 +103,7 @@ public class HelixConfigScope {
    * this is the monitoring config name if type is MONITORING, or null otherwise
    */
   final String _monitoringConfigName;
+  final String _alertConfigName;
 
   final String _resourceName;
 
@@ -156,6 +160,13 @@ public class HelixConfigScope {
       _monitoringConfigName = null;
     }
 
+    // init alert config name
+    if (type == ConfigScopeProperty.ALERT && _isFullKey) {
+      _alertConfigName = zkPathKeys.get(1);
+    } else {
+      _alertConfigName = null;
+    }
+
     _zkPath = template.instantiate(type, zkPathKeys.toArray(new String[0]));
     _mapKey = mapKey;
   }
@@ -201,6 +212,14 @@ public class HelixConfigScope {
   }
 
   /**
+   * Get the alert config name if exists
+   * @return the alert config name if the type is ALERT, or null otherwise
+   */
+  public String getAlertConfigName() {
+    return _alertConfigName;
+  }
+
+  /**
    * Get the path to the corresponding ZNode
    * @return a Zookeeper path
    */

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/main/java/org/apache/helix/model/Message.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/model/Message.java b/helix-core/src/main/java/org/apache/helix/model/Message.java
index d465a80..5dbb57c 100644
--- a/helix-core/src/main/java/org/apache/helix/model/Message.java
+++ b/helix-core/src/main/java/org/apache/helix/model/Message.java
@@ -60,7 +60,8 @@ public class Message extends HelixProperty {
     CONTROLLER_MSG,
     TASK_REPLY,
     NO_OP,
-    PARTICIPANT_ERROR_REPORT
+    PARTICIPANT_ERROR_REPORT,
+    ALERT;
   };
 
   /**
@@ -92,7 +93,8 @@ public class Message extends HelixProperty {
     STATE_MODEL_FACTORY_NAME,
     BUCKET_SIZE,
     PARENT_MSG_ID, // used for group message mode
-    INNER_MESSAGE
+    INNER_MESSAGE,
+    ALERT_NAME;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
index 1d02275..bf95223 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
@@ -39,12 +39,9 @@ import org.apache.commons.cli.ParseException;
 import org.apache.helix.HelixAdmin;
 import org.apache.helix.HelixConstants.StateModelToken;
 import org.apache.helix.HelixException;
-import org.apache.helix.PropertyKey.Builder;
 import org.apache.helix.ZNRecord;
 import org.apache.helix.manager.zk.ZKHelixAdmin;
-import org.apache.helix.manager.zk.ZKHelixDataAccessor;
 import org.apache.helix.manager.zk.ZNRecordSerializer;
-import org.apache.helix.manager.zk.ZkBaseDataAccessor;
 import org.apache.helix.manager.zk.ZkClient;
 import org.apache.helix.model.ClusterConstraints;
 import org.apache.helix.model.ClusterConstraints.ConstraintType;
@@ -55,7 +52,6 @@ import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
 import org.apache.helix.model.IdealState;
 import org.apache.helix.model.IdealState.RebalanceMode;
 import org.apache.helix.model.InstanceConfig;
-import org.apache.helix.model.LiveInstance;
 import org.apache.helix.model.StateModelDefinition;
 import org.apache.helix.model.builder.ConstraintItemBuilder;
 import org.apache.helix.model.builder.HelixConfigScopeBuilder;
@@ -64,7 +60,7 @@ import org.apache.helix.util.ZKClientPool;
 import org.apache.log4j.Logger;
 
 public class ClusterSetup {
-  private static Logger logger = Logger.getLogger(ClusterSetup.class);
+  private static Logger LOG = Logger.getLogger(ClusterSetup.class);
   public static final String zkServerAddress = "zkSvr";
 
   // List info about the cluster / resource / Instances
@@ -134,20 +130,19 @@ public class ClusterSetup {
   public static final String removeConstraint = "removeConstraint";
 
   static Logger _logger = Logger.getLogger(ClusterSetup.class);
-  String _zkServerAddress;
-  ZkClient _zkClient;
-  HelixAdmin _admin;
+  final HelixAdmin _admin;
 
   public ClusterSetup(String zkServerAddress) {
-    _zkServerAddress = zkServerAddress;
-    _zkClient = ZKClientPool.getZkClient(_zkServerAddress);
-    _admin = new ZKHelixAdmin(_zkClient);
+    ZkClient zkClient = ZKClientPool.getZkClient(zkServerAddress);
+    _admin = new ZKHelixAdmin(zkClient);
   }
 
   public ClusterSetup(ZkClient zkClient) {
-    _zkServerAddress = zkClient.getServers();
-    _zkClient = zkClient;
-    _admin = new ZKHelixAdmin(_zkClient);
+    _admin = new ZKHelixAdmin(zkClient);
+  }
+
+  public ClusterSetup(HelixAdmin admin) {
+    _admin = admin;
   }
 
   public void addCluster(String clusterName, boolean overwritePrevious) {
@@ -238,101 +233,12 @@ public class ClusterSetup {
   }
 
   public void dropInstanceFromCluster(String clusterName, String instanceId) {
-    ZKHelixDataAccessor accessor =
-        new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
-    Builder keyBuilder = accessor.keyBuilder();
-
     InstanceConfig instanceConfig = toInstanceConfig(instanceId);
-    instanceId = instanceConfig.getInstanceName();
-
-    // ensure node is stopped
-    LiveInstance liveInstance = accessor.getProperty(keyBuilder.liveInstance(instanceId));
-    if (liveInstance != null) {
-      throw new HelixException("Can't drop " + instanceId + ", please stop " + instanceId
-          + " before drop it");
-    }
-
-    InstanceConfig config = accessor.getProperty(keyBuilder.instanceConfig(instanceId));
-    if (config == null) {
-      String error = "Node " + instanceId + " does not exist, cannot drop";
-      _logger.warn(error);
-      throw new HelixException(error);
-    }
-
-    // ensure node is disabled, otherwise fail
-    if (config.getInstanceEnabled()) {
-      String error = "Node " + instanceId + " is enabled, cannot drop";
-      _logger.warn(error);
-      throw new HelixException(error);
-    }
-    _admin.dropInstance(clusterName, config);
+    _admin.dropInstance(clusterName, instanceConfig);
   }
 
   public void swapInstance(String clusterName, String oldInstanceName, String newInstanceName) {
-    ZKHelixDataAccessor accessor =
-        new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
-    Builder keyBuilder = accessor.keyBuilder();
-
-    InstanceConfig oldConfig = accessor.getProperty(keyBuilder.instanceConfig(oldInstanceName));
-    if (oldConfig == null) {
-      String error = "Old instance " + oldInstanceName + " does not exist, cannot swap";
-      _logger.warn(error);
-      throw new HelixException(error);
-    }
-
-    InstanceConfig newConfig = accessor.getProperty(keyBuilder.instanceConfig(newInstanceName));
-    if (newConfig == null) {
-      String error = "New instance " + newInstanceName + " does not exist, cannot swap";
-      _logger.warn(error);
-      throw new HelixException(error);
-    }
-
-    // ensure old instance is disabled, otherwise fail
-    if (oldConfig.getInstanceEnabled()) {
-      String error =
-          "Old instance " + oldInstanceName + " is enabled, it need to be disabled and turned off";
-      _logger.warn(error);
-      throw new HelixException(error);
-    }
-    // ensure old instance is down, otherwise fail
-    List<String> liveInstanceNames = accessor.getChildNames(accessor.keyBuilder().liveInstances());
-
-    if (liveInstanceNames.contains(oldInstanceName)) {
-      String error =
-          "Old instance " + oldInstanceName + " is still on, it need to be disabled and turned off";
-      _logger.warn(error);
-      throw new HelixException(error);
-    }
-
-    dropInstanceFromCluster(clusterName, oldInstanceName);
-
-    List<IdealState> existingIdealStates =
-        accessor.getChildValues(accessor.keyBuilder().idealStates());
-    for (IdealState idealState : existingIdealStates) {
-      swapInstanceInIdealState(idealState, oldInstanceName, newInstanceName);
-      accessor.setProperty(
-          accessor.keyBuilder().idealStates(idealState.getResourceId().stringify()), idealState);
-    }
-  }
-
-  void swapInstanceInIdealState(IdealState idealState, String oldInstance, String newInstance) {
-    for (String partition : idealState.getRecord().getMapFields().keySet()) {
-      Map<String, String> valMap = idealState.getRecord().getMapField(partition);
-      if (valMap.containsKey(oldInstance)) {
-        valMap.put(newInstance, valMap.get(oldInstance));
-        valMap.remove(oldInstance);
-      }
-    }
-
-    for (String partition : idealState.getRecord().getListFields().keySet()) {
-      List<String> valList = idealState.getRecord().getListField(partition);
-      for (int i = 0; i < valList.size(); i++) {
-        if (valList.get(i).equals(oldInstance)) {
-          valList.remove(i);
-          valList.add(i, newInstance);
-        }
-      }
-    }
+    _admin.swapInstance(clusterName, oldInstanceName, newInstanceName);
   }
 
   public HelixAdmin getClusterManagementTool() {
@@ -590,18 +496,7 @@ public class ClusterSetup {
   }
 
   @SuppressWarnings("static-access")
-  private static Options constructCommandLineOptions() {
-    Option helpOption =
-        OptionBuilder.withLongOpt(help).withDescription("Prints command-line options info")
-            .create();
-
-    Option zkServerOption =
-        OptionBuilder.withLongOpt(zkServerAddress).withDescription("Provide zookeeper address")
-            .create();
-    zkServerOption.setArgs(1);
-    zkServerOption.setRequired(true);
-    zkServerOption.setArgName("ZookeeperServerAddress(Required)");
-
+  public static OptionGroup constructOptionGroup() {
     Option listClustersOption =
         OptionBuilder.withLongOpt(listClusters).withDescription("List existing clusters").create();
     listClustersOption.setArgs(0);
@@ -791,7 +686,7 @@ public class ClusterSetup {
     partitionInfoOption.setArgName("clusterName resourceName partitionName");
 
     Option enableInstanceOption =
-        OptionBuilder.withLongOpt(enableInstance).withDescription("Enable/disable a Instance")
+        OptionBuilder.withLongOpt(enableInstance).withDescription("Enable/disable an instance")
             .create();
     enableInstanceOption.setArgs(3);
     enableInstanceOption.setRequired(false);
@@ -985,10 +880,26 @@ public class ClusterSetup {
     group.addOption(removeInstanceTagOption);
     group.addOption(instanceGroupTagOption);
 
+    return group;
+  }
+
+  @SuppressWarnings("static-access")
+  private static Options constructCommandLineOptions() {
+    Option helpOption =
+        OptionBuilder.withLongOpt(help).withDescription("Prints command-line options info")
+            .create();
+
+    Option zkServerOption =
+        OptionBuilder.withLongOpt(zkServerAddress).withDescription("Provide zookeeper address")
+            .create();
+    zkServerOption.setArgs(1);
+    zkServerOption.setRequired(true);
+    zkServerOption.setArgName("ZookeeperServerAddress(Required)");
+
     Options options = new Options();
     options.addOption(helpOption);
     options.addOption(zkServerOption);
-    options.addOptionGroup(group);
+    options.addOptionGroup(constructOptionGroup());
     return options;
   }
 
@@ -1021,7 +932,18 @@ public class ClusterSetup {
       System.exit(1);
     }
 
+    if (cmd.hasOption(help)) {
+      printUsage(cliOptions);
+      return 0;
+    }
+
     ClusterSetup setupTool = new ClusterSetup(cmd.getOptionValue(zkServerAddress));
+    processCommandLineArgs(setupTool, cmd);
+    return 0;
+  }
+
+  public static int processCommandLineArgs(ClusterSetup setupTool, CommandLine cmd)
+      throws IOException {
 
     if (cmd.hasOption(addCluster)) {
       String clusterName = cmd.getOptionValue(addCluster);
@@ -1419,11 +1341,6 @@ public class ClusterSetup {
       String instanceName = cmd.getOptionValues(removeInstanceTag)[1];
       String tag = cmd.getOptionValues(removeInstanceTag)[2];
       setupTool.getClusterManagementTool().removeInstanceTag(clusterName, instanceName, tag);
-    }
-    // help option
-    else if (cmd.hasOption(help)) {
-      printUsage(cliOptions);
-      return 0;
     } else if (cmd.hasOption(addResourceProperty)) {
       String clusterName = cmd.getOptionValues(addResourceProperty)[0];
       String resourceName = cmd.getOptionValues(addResourceProperty)[1];

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/main/java/org/apache/helix/util/StatusUpdateUtil.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/util/StatusUpdateUtil.java b/helix-core/src/main/java/org/apache/helix/util/StatusUpdateUtil.java
index d0a57f1..e05ad96 100644
--- a/helix-core/src/main/java/org/apache/helix/util/StatusUpdateUtil.java
+++ b/helix-core/src/main/java/org/apache/helix/util/StatusUpdateUtil.java
@@ -489,7 +489,7 @@ public class StatusUpdateUtil {
       // ../{sessionId}/{subPath}
       // accessor.setProperty(PropertyType.ERRORS_CONTROLLER, record,
       // statusUpdateSubPath);
-      accessor.setProperty(keyBuilder.controllerTaskError(statusUpdateSubPath), new Error(record));
+      accessor.updateProperty(keyBuilder.controllerTaskError(statusUpdateSubPath), new Error(record));
     } else {
       // accessor.updateProperty(PropertyType.ERRORS,
       // record,

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/test/java/org/apache/helix/controller/alert/TestAlertAction.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/controller/alert/TestAlertAction.java b/helix-core/src/test/java/org/apache/helix/controller/alert/TestAlertAction.java
new file mode 100644
index 0000000..9a6d636
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/controller/alert/TestAlertAction.java
@@ -0,0 +1,95 @@
+package org.apache.helix.controller.alert;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestAlertAction {
+  @Test
+  public void testBasic() throws Exception {
+    AlertAction.Builder builder = new AlertAction.Builder();
+
+    AlertAction action =
+        builder.cmd("enableInstance").args("TestCluster", "{node}", "false").build();
+    Assert.assertNotNull(action.getCliArgs());
+    Assert.assertEquals(action.getCliArgs().length, 4);
+    Assert.assertEquals(action.getCliArgs()[0], "enableInstance");
+    Assert.assertEquals(action.getCliArgs()[1], "TestCluster");
+    Assert.assertEquals(action.getCliArgs()[2], "{node}");
+    Assert.assertEquals(action.getCliArgs()[3], "false");
+    Assert.assertEquals(action.toString(), "(enableInstance)(TestCluster {node} false)");
+
+    action = AlertAction.from("(enableInstance)(TestCluster {node} true)");
+    Assert.assertNotNull(action.getCliArgs());
+    Assert.assertEquals(action.getCliArgs().length, 4);
+    Assert.assertEquals(action.getCliArgs()[0], "enableInstance");
+    Assert.assertEquals(action.getCliArgs()[1], "TestCluster");
+    Assert.assertEquals(action.getCliArgs()[2], "{node}");
+    Assert.assertEquals(action.getCliArgs()[3], "true");
+  }
+
+  @Test
+  public void testInvalidActionStr() {
+    try {
+      AlertAction.from("(invalidCmd)(TestCluster {node} true)");
+      Assert.fail("Should fail on invalid HelixAdminCmd");
+    } catch (IllegalArgumentException e) {
+      // ok
+    }
+
+    try {
+      AlertAction.from("(enableInstance)(TestCluster {invalidScope} true)");
+      Assert.fail("Should fail on invalid alertScope");
+    } catch (IllegalArgumentException e) {
+      // ok
+    }
+
+    try {
+      AlertAction.from("(enableInstance)()(TestCluster {node} true)");
+      Assert.fail("Should fail on invalid alertAction format");
+    } catch (IllegalArgumentException e) {
+      // ok
+    }
+
+  }
+
+  @Test
+  public void testInvaidConstructorArgument() {
+    try {
+      new AlertAction(null, new String[] {
+          "TestCluster", "{node}", "true)"
+      });
+      Assert.fail("Should fail on null HelixAdminCmd");
+    } catch (NullPointerException e) {
+      // ok
+    }
+
+    try {
+      new AlertAction("enableInstance", new String[] {
+          "TestCluster", "{invalidScope}", "true)"
+      });
+      Assert.fail("Should fail on invalid alert scope");
+    } catch (IllegalArgumentException e) {
+      // ok
+    }
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/test/java/org/apache/helix/controller/alert/TestAlertName.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/controller/alert/TestAlertName.java b/helix-core/src/test/java/org/apache/helix/controller/alert/TestAlertName.java
new file mode 100644
index 0000000..1478be5
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/controller/alert/TestAlertName.java
@@ -0,0 +1,110 @@
+package org.apache.helix.controller.alert;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.helix.api.id.ClusterId;
+import org.apache.helix.api.id.ParticipantId;
+import org.apache.helix.api.id.PartitionId;
+import org.apache.helix.api.id.ResourceId;
+import org.apache.helix.controller.alert.AlertScope.AlertScopeField;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestAlertName {
+
+  @Test
+  public void testBasic() {
+    AlertName.Builder builder = new AlertName.Builder();
+    AlertName name =
+        builder.cluster(ClusterId.from("TestCluster")).node(ParticipantId.from("localhost_12918"))
+            .resource(ResourceId.from("TestDB")).partitionId(PartitionId.from("TestDB_0"))
+            .metric("latency95").largerThan("1000").build();
+    Assert.assertEquals(name.toString(),
+        "(TestCluster.%.localhost_12918.TestDB.TestDB_0)(latency95)>(1000)");
+
+    name = AlertName.from("(TestCluster.%.localhost_12918.TestDB.TestDB_0)(latency95)>(1000)");
+    Assert.assertEquals(name._scope.getClusterId(), ClusterId.from("TestCluster"));
+    Assert.assertNull(name._scope.get(AlertScopeField.tenant));
+    Assert.assertEquals(name._scope.get(AlertScopeField.node),
+        ParticipantId.from("localhost_12918"));
+    Assert.assertEquals(name._scope.get(AlertScopeField.resource), ParticipantId.from("TestDB"));
+    Assert.assertEquals(name._scope.get(AlertScopeField.partition), ParticipantId.from("TestDB_0"));
+    Assert.assertEquals(name._comparator, AlertComparator.LARGER);
+    Assert.assertEquals(name._metric, "latency95");
+    Assert.assertEquals(name._value, "1000");
+  }
+
+  @Test
+  public void testInvalidAlertNameString() {
+    try {
+      AlertName.from("(TestCluster.%.localhost_12918.TestDB.TestDB_0)(latency95)><(1000)");
+      Assert.fail("Should fail on invalid comparator ><");
+    } catch (IllegalArgumentException e) {
+      // ok
+    }
+
+    try {
+      AlertName.from("(TestCluster.%.localhost_12918.TestDB.TestDB_0)()(latency95)>()(1000)");
+      Assert.fail("Should fail on invalid alert name format");
+    } catch (IllegalArgumentException e) {
+      // ok
+    }
+
+    try {
+      AlertName.from("(TestCluster.%.localhost_12918.TestDB.TestDB_0)>(1000)");
+      Assert.fail("Should fail on missing metric field");
+    } catch (IllegalArgumentException e) {
+      // ok
+    }
+
+    try {
+      AlertName.from("(TestCluster.%.localhost_12918.TestDB.TestDB_0.blabla)(latency95)>(1000)");
+      Assert.fail("Should fail on invalid too many scope arguments");
+    } catch (IllegalArgumentException e) {
+      // ok
+    }
+  }
+
+  @Test
+  public void testMatch() {
+    AlertName.Builder builder;
+    AlertName matchingName;
+    boolean match;
+
+    builder = new AlertName.Builder();
+    AlertName name =
+        builder.cluster(ClusterId.from("TestCluster")).resource(ResourceId.from("TestDB"))
+            .metric("latency95").largerThan("1000").build();
+
+    builder = new AlertName.Builder();
+    matchingName =
+        builder.cluster(ClusterId.from("TestCluster")).node(ParticipantId.from("localhost_12918"))
+            .resource(ResourceId.from("TestDB")).metric("latency95").largerThan("1000").build();
+    match = name.match(matchingName);
+    Assert.assertTrue(match);
+
+    builder = new AlertName.Builder();
+    matchingName =
+        builder.cluster(ClusterId.from("TestCluster")).node(ParticipantId.from("localhost_12918"))
+            .resource(ResourceId.from("MyDB")).metric("latency95").largerThan("1000").build();
+    match = name.match(matchingName);
+    Assert.assertFalse(match);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/test/java/org/apache/helix/controller/alert/TestAlertScope.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/controller/alert/TestAlertScope.java b/helix-core/src/test/java/org/apache/helix/controller/alert/TestAlertScope.java
new file mode 100644
index 0000000..45e8530
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/controller/alert/TestAlertScope.java
@@ -0,0 +1,78 @@
+package org.apache.helix.controller.alert;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.helix.api.id.ClusterId;
+import org.apache.helix.api.id.ParticipantId;
+import org.apache.helix.api.id.ResourceId;
+import org.apache.helix.controller.alert.AlertScope.AlertScopeField;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestAlertScope {
+
+  @Test
+  public void testBasic() {
+    AlertScope scope =
+        new AlertScope(ClusterId.from("TestCluster"), null, ParticipantId.from("localhost_12918"),
+            ResourceId.from("TestDB"), null);
+
+    Assert.assertEquals(scope.getClusterId(), ClusterId.from("TestCluster"));
+    Assert.assertEquals(scope.get(AlertScopeField.node), ParticipantId.from("localhost_12918"));
+    Assert.assertEquals(scope.get(AlertScopeField.resource), ResourceId.from("TestDB"));
+    Assert.assertEquals(scope.toString(), "TestCluster.%.localhost_12918.TestDB.%");
+
+    AlertScope matchingScope = new AlertScope("TestCluster", "Tenant1", "localhost_12918");
+    boolean match = scope.match(matchingScope);
+    Assert.assertTrue(match);
+
+  }
+
+  @Test
+  public void testEmptyClusterId() {
+    AlertScope scope = new AlertScope((String[]) null);
+    Assert.assertEquals(scope.toString(), "%.%.%.%.%");
+
+    scope = new AlertScope(new String[] {});
+    Assert.assertEquals(scope.toString(), "%.%.%.%.%");
+
+    scope = new AlertScope(null, "Tenant1", "localhost_12918");
+    Assert.assertEquals(scope.toString(), "%.Tenant1.localhost_12918.%.%");
+  }
+
+  @Test
+  public void testWildcard() {
+    AlertScope scope = new AlertScope("%");
+    Assert.assertEquals(scope.toString(), "%.%.%.%.%");
+
+    scope = new AlertScope("TestCluster", "%", "localhost_12918");
+    Assert.assertEquals(scope.toString(), "TestCluster.%.localhost_12918.%.%");
+  }
+
+  @Test
+  public void testTooManyArguments() {
+    try {
+      new AlertScope("TestCluster", null, "localhost_12918", "TestDB", "TestDB_0", "blabla");
+      Assert.fail("Should fail on too many arguments");
+    } catch (IllegalArgumentException e) {
+      // ok
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/test/java/org/apache/helix/manager/zk/TestDefaultAlertMsgHandler.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestDefaultAlertMsgHandler.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestDefaultAlertMsgHandler.java
new file mode 100644
index 0000000..b5a7049
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestDefaultAlertMsgHandler.java
@@ -0,0 +1,182 @@
+package org.apache.helix.manager.zk;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Date;
+
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.api.id.ClusterId;
+import org.apache.helix.api.id.MessageId;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.messaging.handling.MessageHandlerFactory;
+import org.apache.helix.controller.alert.AlertAction;
+import org.apache.helix.model.AlertConfig;
+import org.apache.helix.controller.alert.AlertName;
+import org.apache.helix.model.Error;
+import org.apache.helix.model.InstanceConfig;
+import org.apache.helix.model.Message.MessageType;
+import org.apache.helix.model.Message;
+import org.apache.helix.tools.ClusterStateVerifier;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestDefaultAlertMsgHandler extends ZkUnitTestBase {
+
+  @Test
+  public void testBasic() throws Exception {
+    String className = TestHelper.getTestClassName();
+    String methodName = TestHelper.getTestMethodName();
+    String clusterName = className + "_" + methodName;
+    int n = 2;
+
+    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
+
+    TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, // participant port
+        "localhost", // participant name prefix
+        "TestDB", // resource name prefix
+        1, // resources
+        32, // partitions per resource
+        n, // number of nodes
+        2, // replicas
+        "MasterSlave", true); // do rebalance
+
+    ClusterControllerManager controller =
+        new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
+    MessageHandlerFactory fty = new DefaultAlertMsgHandlerFactory();
+    controller.getMessagingService().registerMessageHandlerFactory(fty.getMessageType(), fty);
+    controller.syncStart();
+
+    // start participants
+    MockParticipantManager[] participants = new MockParticipantManager[n];
+    for (int i = 0; i < n; i++) {
+      String instanceName = "localhost_" + (12918 + i);
+
+      participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
+      participants[i].syncStart();
+    }
+
+    boolean result =
+        ClusterStateVerifier
+            .verifyByZkCallback(new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR,
+                clusterName));
+    Assert.assertTrue(result);
+
+    // add alert config
+    final HelixDataAccessor accessor = controller.getHelixDataAccessor();
+    AlertConfig config = new AlertConfig("default");
+    AlertName name =
+        new AlertName.Builder().cluster(ClusterId.from(clusterName)).metric("latency95")
+            .largerThan("1000").build();
+    AlertAction action =
+        new AlertAction.Builder().cmd("enableInstance")
+            .args("{cluster}", "{node}", "false").build();
+    config.putConfig(name, action);
+    final PropertyKey.Builder keyBuilder = new PropertyKey.Builder(clusterName);
+    accessor.setProperty(keyBuilder.alertConfig("default"), config);
+
+    Message alertMsg;
+
+    // test send an alert message that matches the config
+    alertMsg = new Message(MessageType.ALERT, MessageId.from("msg_1"));
+    alertMsg.setAttribute(Message.Attributes.ALERT_NAME,
+        String.format("(%s.%s.%s)(latency95)>(1000)", clusterName, "tenant1", "localhost_12918"));
+    alertMsg.setTgtSessionId(controller.getSessionId());
+    alertMsg.setTgtName("controller");
+    accessor.setProperty(keyBuilder.controllerMessage(alertMsg.getId()), alertMsg);
+
+    // verify localhost_12918 is disabled
+    result = TestHelper.verify(new TestHelper.Verifier() {
+
+      @Override
+      public boolean verify() throws Exception {
+        InstanceConfig config = accessor.getProperty(keyBuilder.instanceConfig("localhost_12918"));
+        return !config.getInstanceEnabled();
+      }
+    }, 10 * 1000);
+    Assert.assertTrue(result);
+
+    // test send an alert message that doesn't match any config
+    alertMsg = new Message(MessageType.ALERT, MessageId.from("msg_2"));
+    alertMsg.setAttribute(Message.Attributes.ALERT_NAME,
+        String.format("(%s.%s.%s)(errorRatio)>(0.1)", clusterName, "tenant1", "localhost_12918"));
+    alertMsg.setTgtSessionId(controller.getSessionId());
+    alertMsg.setTgtName("controller");
+    accessor.setProperty(keyBuilder.controllerMessage(alertMsg.getId()), alertMsg);
+    result = containsCtrlErrMsg(accessor, "msg_2");
+    Assert.assertTrue(result);
+
+    // test send an invalid alert message that doesn't have {node} scope
+    alertMsg = new Message(MessageType.ALERT, MessageId.from("msg_3"));
+    alertMsg.setAttribute(Message.Attributes.ALERT_NAME,
+        String.format("(%s.%s.%%.%s)(latency95)>(1000)", clusterName, "tenant1", "TestDB"));
+    alertMsg.setTgtSessionId(controller.getSessionId());
+    alertMsg.setTgtName("controller");
+    accessor.setProperty(keyBuilder.controllerMessage(alertMsg.getId()), alertMsg);
+    result = containsCtrlErrMsg(accessor, "msg_3");
+    Assert.assertTrue(result);
+
+    // clean up
+    controller.syncStop();
+    for (int i = 0; i < n; i++) {
+      participants[i].syncStop();
+    }
+
+    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
+
+  }
+
+  /**
+   * verify we have an error entry for msgId in CONTROLLER/ERRORS/ALERT
+   * @param msgId
+   * @return
+   */
+  private boolean containsCtrlErrMsg(final HelixDataAccessor accessor, final String msgId)
+      throws Exception {
+    boolean result = TestHelper.verify(new TestHelper.Verifier() {
+      final PropertyKey.Builder keyBuilder = accessor.keyBuilder();
+
+      @Override
+      public boolean verify() throws Exception {
+        Error error =
+            accessor.getProperty(keyBuilder.controllerTaskError(MessageType.ALERT.name()));
+        if (error == null) {
+          return false;
+        }
+
+        for (String key : error.getRecord().getMapFields().keySet()) {
+          if (!key.startsWith("HELIX_ERROR")) {
+            continue;
+          }
+          String tmpMsgId = error.getRecord().getMapField(key).get("MSG_ID");
+          if (tmpMsgId != null && tmpMsgId.equals(msgId)) {
+            return true;
+          }
+        }
+        return false;
+      }
+    }, 10 * 1000);
+
+    return result;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
index 89a947f..ed2f0f7 100644
--- a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
@@ -100,6 +100,7 @@ public class TestZkHelixAdmin extends ZkUnitTestBase {
     config = tool.getInstanceConfig(clusterName, "host1_9999");
     AssertJUnit.assertEquals(config.getId(), "host1_9999");
 
+    tool.enableInstance(clusterName, "host1_9999", false);
     tool.dropInstance(clusterName, config);
     try {
       tool.getInstanceConfig(clusterName, "host1_9999");

http://git-wip-us.apache.org/repos/asf/helix/blob/b182690f/helix-core/src/test/java/org/apache/helix/model/TestAlertConfig.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/model/TestAlertConfig.java b/helix-core/src/test/java/org/apache/helix/model/TestAlertConfig.java
new file mode 100644
index 0000000..b2ce9e1
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/model/TestAlertConfig.java
@@ -0,0 +1,71 @@
+package org.apache.helix.model;
+
+import org.apache.helix.api.id.ClusterId;
+import org.apache.helix.api.id.ParticipantId;
+import org.apache.helix.api.id.ResourceId;
+import org.apache.helix.controller.alert.AlertAction;
+import org.apache.helix.controller.alert.AlertName;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+public class TestAlertConfig {
+  @Test
+  public void testBasic() {
+    AlertConfig config = new AlertConfig("default");
+    AlertName name =
+        new AlertName.Builder().cluster(ClusterId.from("TestCluster")).metric("latency95")
+            .largerThan("1000").build();
+    AlertAction action =
+        new AlertAction.Builder().cmd("enableInstance")
+            .args("{cluster}", "{node}", "false").build();
+    config.putConfig(name, action);
+
+    AlertName matchingName =
+        new AlertName.Builder().cluster(ClusterId.from("TestCluster"))
+            .node(ParticipantId.from("localhost_10001")).metric("latency95").largerThan("1000")
+            .build();
+    AlertAction matchingAction = config.findAlertAction(matchingName);
+
+    Assert.assertNotNull(matchingAction);
+    Assert.assertEquals(matchingAction.toString(),
+        "(enableInstance)(TestCluster localhost_10001 false)");
+  }
+
+  @Test
+  public void testFail() {
+    AlertConfig config = new AlertConfig("default");
+    AlertName name =
+        new AlertName.Builder().cluster(ClusterId.from("TestCluster")).metric("latency95")
+            .largerThan("1000").build();
+    AlertAction action =
+        new AlertAction.Builder().cmd("enableInstance")
+            .args("{cluster}", "{node}", "false").build();
+    config.putConfig(name, action);
+
+    AlertName matchingName =
+        new AlertName.Builder().cluster(ClusterId.from("TestCluster"))
+            .resource(ResourceId.from("TestDB")).metric("latency95").largerThan("1000").build();
+    AlertAction matchingAction = config.findAlertAction(matchingName);
+    Assert.assertNull(matchingAction, "Should fail on null scope value for {node}");
+
+  }
+}
\ No newline at end of file


Mime
View raw message