helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l...@apache.org
Subject [4/4] helix git commit: [HELIX-622] Add new resource configuration option to allow resource to disable emmiting monitoring bean.
Date Thu, 16 Jun 2016 23:36:13 GMT
[HELIX-622] Add new resource configuration option to allow resource to disable emmiting monitoring
bean.


Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/24096019
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/24096019
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/24096019

Branch: refs/heads/helix-0.6.x
Commit: 24096019375e13c439ec6bfa83088ba7c25ffaf9
Parents: 6b6bb8f
Author: Lei Xia <lxia@linkedin.com>
Authored: Fri Jan 8 17:14:00 2016 -0800
Committer: Lei Xia <lxia@linkedin.com>
Committed: Wed Apr 13 10:43:23 2016 -0700

----------------------------------------------------------------------
 .../java/org/apache/helix/HelixConstants.java   |   5 +-
 .../main/java/org/apache/helix/PropertyKey.java |   6 +-
 .../stages/BestPossibleStateCalcStage.java      |   1 +
 .../controller/stages/ClusterDataCache.java     |  37 +++++--
 .../stages/ExternalViewComputeStage.java        |  20 ++--
 .../org/apache/helix/model/ResourceConfig.java  | 106 ++++++++++++++++++
 .../mbeans/TestDisableResourceMbean.java        | 109 +++++++++++++++++++
 7 files changed, 264 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/helix/blob/24096019/helix-core/src/main/java/org/apache/helix/HelixConstants.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/HelixConstants.java b/helix-core/src/main/java/org/apache/helix/HelixConstants.java
index 6b6287c..5318fa9 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixConstants.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixConstants.java
@@ -44,8 +44,9 @@ public interface HelixConstants {
   }
 
   enum ClusterConfigType {
-    HELIX_DISABLE_PIPELINE_TRIGGERS
+    HELIX_DISABLE_PIPELINE_TRIGGERS,
+    DISABLE_FULL_AUTO // override all resources in the cluster to use SEMI-AUTO instead of
FULL-AUTO
   }
 
-  static final String DEFAULT_STATE_MODEL_FACTORY = "DEFAULT";
+  String DEFAULT_STATE_MODEL_FACTORY = "DEFAULT";
 }

http://git-wip-us.apache.org/repos/asf/helix/blob/24096019/helix-core/src/main/java/org/apache/helix/PropertyKey.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyKey.java b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
index 663e831..33355f1 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyKey.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
@@ -50,8 +50,10 @@ import org.apache.helix.model.LeaderHistory;
 import org.apache.helix.model.LiveInstance;
 import org.apache.helix.model.Message;
 import org.apache.helix.model.PauseSignal;
+import org.apache.helix.model.ResourceConfig;
 import org.apache.helix.model.StateModelDefinition;
 import org.apache.helix.model.StatusUpdate;
+import org.apache.helix.tools.YAMLClusterSetup;
 import org.apache.log4j.Logger;
 
 /**
@@ -212,7 +214,7 @@ public class PropertyKey {
      * @return {@link PropertyKey}
      */
     public PropertyKey resourceConfigs() {
-      return new PropertyKey(CONFIGS, ConfigScopeProperty.RESOURCE, HelixProperty.class,
+      return new PropertyKey(CONFIGS, ConfigScopeProperty.RESOURCE, ResourceConfig.class,
           _clusterName, ConfigScopeProperty.RESOURCE.toString());
     }
 
@@ -222,7 +224,7 @@ public class PropertyKey {
      * @return {@link PropertyKey}
      */
     public PropertyKey resourceConfig(String resourceName) {
-      return new PropertyKey(CONFIGS, ConfigScopeProperty.RESOURCE, HelixProperty.class,
+      return new PropertyKey(CONFIGS, ConfigScopeProperty.RESOURCE, ResourceConfig.class,
           _clusterName, ConfigScopeProperty.RESOURCE.toString(), resourceName);
     }
 

http://git-wip-us.apache.org/repos/asf/helix/blob/24096019/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
b/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
index 9a9767e..f12b6e5 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
@@ -21,6 +21,7 @@ package org.apache.helix.controller.stages;
 
 import java.util.Map;
 
+import org.apache.helix.HelixConstants;
 import org.apache.helix.HelixManager;
 import org.apache.helix.controller.pipeline.AbstractBaseStage;
 import org.apache.helix.controller.pipeline.StageException;

http://git-wip-us.apache.org/repos/asf/helix/blob/24096019/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
b/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
index 7144077..fde4959 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
@@ -39,6 +39,7 @@ import org.apache.helix.model.IdealState;
 import org.apache.helix.model.InstanceConfig;
 import org.apache.helix.model.LiveInstance;
 import org.apache.helix.model.Message;
+import org.apache.helix.model.ResourceConfig;
 import org.apache.helix.model.StateModelDefinition;
 import org.apache.log4j.Logger;
 
@@ -61,6 +62,8 @@ public class ClusterDataCache {
   Map<String, StateModelDefinition> _stateModelDefMap;
   Map<String, InstanceConfig> _instanceConfigMap;
   Map<String, InstanceConfig> _instanceConfigCacheMap;
+  Map<String, ResourceConfig> _resourceConfigMap;
+  Map<String, ResourceConfig> _resourceConfigCacheMap;
   Map<String, ClusterConstraints> _constraintMap;
   Map<String, Map<String, Map<String, CurrentState>>> _currentStateMap;
   Map<String, Map<String, Message>> _messageMap;
@@ -89,10 +92,15 @@ public class ClusterDataCache {
       _idealStateCacheMap = accessor.getChildValuesMap(keyBuilder.idealStates());
       _liveInstanceCacheMap = accessor.getChildValuesMap(keyBuilder.liveInstances());
       _instanceConfigCacheMap = accessor.getChildValuesMap(keyBuilder.instanceConfigs());
+      _resourceConfigCacheMap = accessor.getChildValuesMap(keyBuilder.resourceConfigs());
     }
     _idealStateMap = Maps.newHashMap(_idealStateCacheMap);
     _liveInstanceMap = Maps.newHashMap(_liveInstanceCacheMap);
     _instanceConfigMap = Maps.newHashMap(_instanceConfigCacheMap);
+    _resourceConfigMap = Maps.newHashMap(_resourceConfigCacheMap);
+
+    _stateModelDefMap = accessor.getChildValuesMap(keyBuilder.stateModelDefs());
+    _constraintMap = accessor.getChildValuesMap(keyBuilder.constraints());
 
     if (LOG.isTraceEnabled()) {
       for (LiveInstance instance : _liveInstanceMap.values()) {
@@ -100,9 +108,6 @@ public class ClusterDataCache {
       }
     }
 
-    _stateModelDefMap = accessor.getChildValuesMap(keyBuilder.stateModelDefs());
-    _constraintMap = accessor.getChildValuesMap(keyBuilder.constraints());
-
     Map<String, Map<String, Message>> msgMap = new HashMap<String, Map<String,
Message>>();
     List<PropertyKey> newMessageKeys = Lists.newLinkedList();
     long purgeSum = 0;
@@ -216,10 +221,9 @@ public class ClusterDataCache {
     LOG.info("END: ClusterDataCache.refresh(), took " + (endTime - startTime) + " ms");
 
     if (LOG.isDebugEnabled()) {
-      int numPaths =
-          _liveInstanceMap.size() + _idealStateMap.size() + _stateModelDefMap.size()
-              + _instanceConfigMap.size() + _constraintMap.size() + newMessageKeys.size()
-              + currentStateKeys.size();
+      int numPaths = _liveInstanceMap.size() + _idealStateMap.size() + _stateModelDefMap.size()
+          + _instanceConfigMap.size() + _resourceConfigMap.size() + _constraintMap.size()
+          + newMessageKeys.size() + currentStateKeys.size();
       LOG.debug("Paths read: " + numPaths);
     }
 
@@ -341,6 +345,24 @@ public class ClusterDataCache {
     return _instanceConfigMap;
   }
 
+  /**
+   * Returns the instance config map
+   *
+   * @return
+   */
+  public Map<String, ResourceConfig> getResourceConfigMap() {
+    return _resourceConfigMap;
+  }
+
+  /**
+   * Returns the instance config map
+   *
+   * @return
+   */
+  public ResourceConfig getResourceConfig(String resource) {
+    return _resourceConfigMap.get(resource);
+  }
+
   public synchronized void setInstanceConfigs(List<InstanceConfig> instanceConfigs)
{
     Map<String, InstanceConfig> instanceConfigMap = Maps.newHashMap();
     for (InstanceConfig instanceConfig : instanceConfigs) {
@@ -424,6 +446,7 @@ public class ClusterDataCache {
     sb.append("idealStateMap:" + _idealStateMap).append("\n");
     sb.append("stateModelDefMap:" + _stateModelDefMap).append("\n");
     sb.append("instanceConfigMap:" + _instanceConfigMap).append("\n");
+    sb.append("resourceConfigMap:" + _resourceConfigMap).append("\n");
     sb.append("messageMap:" + _messageMap).append("\n");
 
     return sb.toString();

http://git-wip-us.apache.org/repos/asf/helix/blob/24096019/helix-core/src/main/java/org/apache/helix/controller/stages/ExternalViewComputeStage.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/ExternalViewComputeStage.java
b/helix-core/src/main/java/org/apache/helix/controller/stages/ExternalViewComputeStage.java
index 1455cd5..d83518d 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/ExternalViewComputeStage.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/ExternalViewComputeStage.java
@@ -44,6 +44,7 @@ import org.apache.helix.model.Message;
 import org.apache.helix.model.Message.MessageType;
 import org.apache.helix.model.Partition;
 import org.apache.helix.model.Resource;
+import org.apache.helix.model.ResourceConfig;
 import org.apache.helix.model.StateModelDefinition;
 import org.apache.helix.model.StatusUpdate;
 import org.apache.helix.monitoring.mbeans.ClusterStatusMonitor;
@@ -104,20 +105,21 @@ public class ExternalViewComputeStage extends AbstractBaseStage {
         }
       }
       // Update cluster status monitor mbean
-      ClusterStatusMonitor clusterStatusMonitor =
-          (ClusterStatusMonitor) event.getAttribute("clusterStatusMonitor");
+      ClusterStatusMonitor clusterStatusMonitor = event.getAttribute("clusterStatusMonitor");
       IdealState idealState = cache._idealStateMap.get(resourceName);
-      if (idealState != null) {
-        if (clusterStatusMonitor != null
-            && !idealState.getStateModelDefRef().equalsIgnoreCase(
-                DefaultSchedulerMessageHandlerFactory.SCHEDULER_TASK_QUEUE)) {
+      ResourceConfig resourceConfig = cache.getResourceConfig(resourceName);
+      if (idealState != null && (resourceConfig == null || !resourceConfig
+          .isMonitoringDisabled())) {
+        if (clusterStatusMonitor != null && !idealState.getStateModelDefRef()
+            .equalsIgnoreCase(DefaultSchedulerMessageHandlerFactory.SCHEDULER_TASK_QUEUE))
{
           StateModelDefinition stateModelDef =
               cache.getStateModelDef(idealState.getStateModelDefRef());
-          clusterStatusMonitor.setResourceStatus(view,
-              cache._idealStateMap.get(view.getResourceName()), stateModelDef);
+          clusterStatusMonitor
+              .setResourceStatus(view, cache._idealStateMap.get(view.getResourceName()),
+                  stateModelDef);
         }
       } else {
-        // Drop the metrics for the dropped resource
+        // Drop the metrics if the resource is dropped, or the MonitorDisabled is changed
to true.
         clusterStatusMonitor.unregisterResource(view.getResourceName());
       }
 

http://git-wip-us.apache.org/repos/asf/helix/blob/24096019/helix-core/src/main/java/org/apache/helix/model/ResourceConfig.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/model/ResourceConfig.java b/helix-core/src/main/java/org/apache/helix/model/ResourceConfig.java
new file mode 100644
index 0000000..98433f5
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/model/ResourceConfig.java
@@ -0,0 +1,106 @@
+package org.apache.helix.model;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.helix.HelixProperty;
+import org.apache.helix.ZNRecord;
+import org.apache.log4j.Logger;
+
+/**
+ * Resource configurations
+ */
+public class ResourceConfig extends HelixProperty {
+  /**
+   * Configurable characteristics of an instance
+   */
+  public enum ResourceConfigProperty {
+    MONITORING_DISABLED, // Resource-level config, do not create Mbean and report any status
for the resource.
+  }
+
+  private static final Logger _logger = Logger.getLogger(ResourceConfig.class.getName());
+
+  /**
+   * Instantiate for a specific instance
+   *
+   * @param resourceId the instance identifier
+   */
+  public ResourceConfig(String resourceId) {
+    super(resourceId);
+  }
+
+  /**
+   * Instantiate with a pre-populated record
+   *
+   * @param record a ZNRecord corresponding to an instance configuration
+   */
+  public ResourceConfig(ZNRecord record) {
+    super(record);
+  }
+
+  /**
+   * Get the value of DisableMonitoring set.
+   *
+   * @return the MonitoringDisabled is true or false
+   */
+  public Boolean isMonitoringDisabled() {
+    return _record.getBooleanField(ResourceConfigProperty.MONITORING_DISABLED.toString(),
false);
+  }
+
+  /**
+   * Set whether to disable monitoring for this resource.
+   *
+   * @param monitoringDisabled whether to disable monitoring for this resource.
+   */
+  public void setMonitoringDisabled(boolean monitoringDisabled) {
+    _record
+        .setBooleanField(ResourceConfigProperty.MONITORING_DISABLED.toString(), monitoringDisabled);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (obj instanceof ResourceConfig) {
+      ResourceConfig that = (ResourceConfig) obj;
+
+      if (this.getId().equals(that.getId())) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return getId().hashCode();
+  }
+
+  /**
+   * Get the name of this resource
+   *
+   * @return the instance name
+   */
+  public String getResourceName() {
+    return _record.getId();
+  }
+
+  @Override
+  public boolean isValid() {
+    return true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/helix/blob/24096019/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestDisableResourceMbean.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestDisableResourceMbean.java
b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestDisableResourceMbean.java
new file mode 100644
index 0000000..6d67fc7
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestDisableResourceMbean.java
@@ -0,0 +1,109 @@
+package org.apache.helix.monitoring.mbeans;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.helix.ConfigAccessor;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.model.HelixConfigScope;
+import org.apache.helix.model.IdealState.RebalanceMode;
+import org.apache.helix.model.ResourceConfig;
+import org.apache.helix.model.builder.HelixConfigScopeBuilder;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import javax.management.MBeanServerConnection;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import java.lang.management.ManagementFactory;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+public class TestDisableResourceMbean extends ZkUnitTestBase {
+  private MBeanServerConnection _mbeanServer = ManagementFactory.getPlatformMBeanServer();
+
+  @Test public void testDisableResourceMonitoring() throws Exception {
+    final int NUM_PARTICIPANTS = 2;
+    String clusterName = TestHelper.getTestClassName() + "_" + TestHelper.getTestMethodName();
+    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
+
+    // Set up cluster
+    TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, // participant port
+        "localhost", // participant name prefix
+        "TestDB", // resource name prefix
+        3, // resources
+        32, // partitions per resource
+        4, // number of nodes
+        1, // replicas
+        "MasterSlave", RebalanceMode.FULL_AUTO, // use FULL_AUTO mode to test node tagging
+        true); // do rebalance
+
+    MockParticipantManager[] participants = new MockParticipantManager[NUM_PARTICIPANTS];
+    for (int i = 0; i < NUM_PARTICIPANTS; i++) {
+      participants[i] =
+          new MockParticipantManager(ZK_ADDR, clusterName, "localhost_" + (12918 + i));
+      participants[i].syncStart();
+    }
+
+    ConfigAccessor configAccessor = new ConfigAccessor(_gZkClient);
+    HelixConfigScope resourceScope =
+        new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.RESOURCE)
+            .forCluster(clusterName).forResource("TestDB1").build();
+    configAccessor
+        .set(resourceScope, ResourceConfig.ResourceConfigProperty.MONITORING_DISABLED.name(),
+            "true");
+
+    resourceScope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.RESOURCE)
+        .forCluster(clusterName).forResource("TestDB2").build();
+    configAccessor
+        .set(resourceScope, ResourceConfig.ResourceConfigProperty.MONITORING_DISABLED.name(),
+            "false");
+
+    ClusterControllerManager controller =
+        new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
+    controller.syncStart();
+
+    Thread.sleep(300);
+
+    // Verify the bean was created for TestDB0, but not for TestDB1.
+    Assert.assertTrue(_mbeanServer.isRegistered(getMbeanName("TestDB0", clusterName)));
+    Assert.assertFalse(_mbeanServer.isRegistered(getMbeanName("TestDB1", clusterName)));
+    Assert.assertTrue(_mbeanServer.isRegistered(getMbeanName("TestDB2", clusterName)));
+
+    controller.syncStop();
+    for (MockParticipantManager participant : participants) {
+      participant.syncStop();
+    }
+    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
+  }
+
+  private ObjectName getMbeanName(String resourceName, String clusterName)
+      throws MalformedObjectNameException {
+    String clusterBeanName =
+        String.format("%s=%s", ClusterStatusMonitor.CLUSTER_DN_KEY, clusterName);
+    String resourceBeanName = String
+        .format("%s,%s=%s", clusterBeanName, ClusterStatusMonitor.RESOURCE_DN_KEY, resourceName);
+    return new ObjectName(
+        String.format("%s: %s", ClusterStatusMonitor.CLUSTER_STATUS_KEY, resourceBeanName));
+  }
+}


Mime
View raw message