helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject git commit: [HELIX-61]Support the concept of roles/groups for nodes -- auto-rebalancing support the group tag
Date Tue, 26 Mar 2013 23:03:42 GMT
Updated Branches:
  refs/heads/master 77108e4fe -> 404f3d802


[HELIX-61]Support the concept of roles/groups for nodes --
auto-rebalancing support the group tag

Project: http://git-wip-us.apache.org/repos/asf/incubator-helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-helix/commit/404f3d80
Tree: http://git-wip-us.apache.org/repos/asf/incubator-helix/tree/404f3d80
Diff: http://git-wip-us.apache.org/repos/asf/incubator-helix/diff/404f3d80

Branch: refs/heads/master
Commit: 404f3d802fc44e4f31dbb86cdc2ac05ad7e3ea0d
Parents: 77108e4
Author: slu2011 <lushi04@gmail.com>
Authored: Tue Mar 26 16:03:31 2013 -0700
Committer: slu2011 <lushi04@gmail.com>
Committed: Tue Mar 26 16:03:31 2013 -0700

----------------------------------------------------------------------
 .../stages/BestPossibleStateCalcStage.java         |    9 ++-
 .../helix/integration/TestAutoRebalance.java       |   54 ++++++++++++++-
 2 files changed, 58 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/404f3d80/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
b/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
index 11557d7..edce407 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
@@ -191,11 +191,14 @@ public class BestPossibleStateCalcStage extends AbstractBaseStage
     Set<String> taggedInstances = new HashSet<String>();
     
     // If there are instances tagged with resource name, use only those instances
-    for(String instanceName : liveInstances)
+    if(idealState.getInstanceGroupTag() != null)
     {
-      if(cache._instanceConfigMap.get(instanceName).containsTag(idealState.getResourceName()))
+      for(String instanceName : liveInstances)
       {
-        taggedInstances.add(instanceName);
+        if(cache._instanceConfigMap.get(instanceName).containsTag(idealState.getInstanceGroupTag()))
+        {
+          taggedInstances.add(instanceName);
+        }
       }
     }
     if(taggedInstances.size() > 0)

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/404f3d80/helix-core/src/test/java/org/apache/helix/integration/TestAutoRebalance.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestAutoRebalance.java
b/helix-core/src/test/java/org/apache/helix/integration/TestAutoRebalance.java
index 53a9f98..bdb6019 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestAutoRebalance.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestAutoRebalance.java
@@ -21,7 +21,9 @@ package org.apache.helix.integration;
 
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.helix.HelixDataAccessor;
 import org.apache.helix.TestHelper;
@@ -34,6 +36,7 @@ import org.apache.helix.manager.zk.ZKHelixDataAccessor;
 import org.apache.helix.manager.zk.ZNRecordSerializer;
 import org.apache.helix.manager.zk.ZkBaseDataAccessor;
 import org.apache.helix.manager.zk.ZkClient;
+import org.apache.helix.model.ExternalView;
 import org.apache.helix.model.IdealState.IdealStateModeProperty;
 import org.apache.helix.tools.ClusterSetup;
 import org.apache.helix.tools.ClusterStateVerifier;
@@ -47,7 +50,8 @@ import org.testng.annotations.Test;
 public class TestAutoRebalance extends ZkStandAloneCMTestBaseWithPropertyServerCheck
 {
   private static final Logger LOG = Logger.getLogger(TestAutoRebalance.class.getName());
-  
+  String db2 = TEST_DB+"2";
+  String _tag = "SSDSSD";
   @BeforeClass
   public void beforeClass() throws Exception
   {
@@ -67,13 +71,26 @@ public class TestAutoRebalance extends ZkStandAloneCMTestBaseWithPropertyServerC
     // setup storage cluster
     _setupTool.addCluster(CLUSTER_NAME, true);
     _setupTool.addResourceToCluster(CLUSTER_NAME, TEST_DB, _PARTITIONS, STATE_MODEL, IdealStateModeProperty.AUTO_REBALANCE+"");
+    
+    _setupTool.addResourceToCluster(CLUSTER_NAME, db2, _PARTITIONS, "OnlineOffline", IdealStateModeProperty.AUTO_REBALANCE+"");
+    
+    
     for (int i = 0; i < NODE_NR; i++)
     {
       String storageNodeName = PARTICIPANT_PREFIX + ":" + (START_PORT + i);
       _setupTool.addInstanceToCluster(CLUSTER_NAME, storageNodeName);
     }
+    
     _setupTool.rebalanceStorageCluster(CLUSTER_NAME, TEST_DB, _replica);
     
+    for (int i = 0; i < 3; i++)
+    {
+      String storageNodeName = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
+      _setupTool.getClusterManagementTool().addInstanceTag(CLUSTER_NAME, storageNodeName,
_tag);
+    }
+
+    _setupTool.rebalanceStorageCluster(CLUSTER_NAME, db2, 1, "ucpx",_tag);
+    
     // start dummy participants
     for (int i = 0; i < NODE_NR; i++)
     {
@@ -105,6 +122,8 @@ public class TestAutoRebalance extends ZkStandAloneCMTestBaseWithPropertyServerC
                                                                               CLUSTER_NAME,
TEST_DB));
 
     Assert.assertTrue(result);
+    
+    
   }
   
   @Test()
@@ -138,6 +157,24 @@ public class TestAutoRebalance extends ZkStandAloneCMTestBaseWithPropertyServerC
         ClusterStateVerifier.verifyByZkCallback(new ExternalViewBalancedVerifier(_zkClient,
                                                                               CLUSTER_NAME,
TEST_DB));
     Assert.assertTrue(result);
+    
+    result =
+        ClusterStateVerifier.verifyByZkCallback(new ExternalViewBalancedVerifier(_zkClient,
+            CLUSTER_NAME, db2));
+    Assert.assertTrue(result);
+    HelixDataAccessor accessor = new ZKHelixDataAccessor( CLUSTER_NAME, new ZkBaseDataAccessor(_zkClient));
+    Builder keyBuilder = accessor.keyBuilder();
+    ExternalView ev = accessor.getProperty(keyBuilder.externalView(db2));
+    Set<String> instancesSet = new HashSet<String>();
+    for(String partitionName : ev.getRecord().getMapFields().keySet())
+    {
+      Map<String, String> assignmentMap = ev.getRecord().getMapField(partitionName);
+      for(String instance : assignmentMap.keySet())
+      {
+        instancesSet.add(instance);
+      }
+    }
+    Assert.assertEquals(instancesSet.size(), 2);
   }
   
   static boolean verifyBalanceExternalView(ZNRecord externalView, int partitionCount, String
masterState, int replica, int instances)
@@ -209,7 +246,20 @@ public class TestAutoRebalance extends ZkStandAloneCMTestBaseWithPropertyServerC
       cache.refresh(accessor);
       String masterValue = cache.getStateModelDef(cache.getIdealState(_resourceName).getStateModelDefRef()).getStatesPriorityList().get(0);
       int replicas = Integer.parseInt(cache.getIdealState(_resourceName).getReplicas());
-      return verifyBalanceExternalView(accessor.getProperty(keyBuilder.externalView(_resourceName)).getRecord(),
numberOfPartitions, masterValue, replicas, cache.getLiveInstances().size());
+      String instanceGroupTag = cache.getIdealState(_resourceName).getInstanceGroupTag();
+      int instances = 0;
+      for(String  liveInstanceName : cache.getLiveInstances().keySet())
+      {
+        if(cache.getInstanceConfigMap().get(liveInstanceName).containsTag(instanceGroupTag))
+        {
+          instances ++;
+        }
+      }
+      if(instances == 0)
+      {
+        instances = cache.getLiveInstances().size();
+      }
+      return verifyBalanceExternalView(accessor.getProperty(keyBuilder.externalView(_resourceName)).getRecord(),
numberOfPartitions, masterValue, replicas, instances);
     }
 
     @Override


Mime
View raw message