helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zzh...@apache.org
Subject [7/7] git commit: merge master
Date Mon, 04 Aug 2014 20:19:24 GMT
merge master


Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/59391888
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/59391888
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/59391888

Branch: refs/heads/master
Commit: 593918880be3b3ab139d7043105ec78fb759ddb2
Parents: 02165c5 8a37057
Author: zzhang <zzhang@apache.org>
Authored: Mon Aug 4 13:18:48 2014 -0700
Committer: zzhang <zzhang@apache.org>
Committed: Mon Aug 4 13:18:48 2014 -0700

----------------------------------------------------------------------
 .../helix/api/accessor/ClusterAccessor.java     |  78 +--
 .../config/BasicRebalancerConfig.java           | 256 ---------
 .../config/CustomRebalancerConfig.java          | 169 ------
 .../config/FullAutoRebalancerConfig.java        |  69 ---
 .../config/PartitionedRebalancerConfig.java     | 523 -------------------
 .../rebalancer/config/RebalancerConfig.java     |  57 +-
 .../config/RebalancerConfigHolder.java          |  15 -
 .../config/ReplicatedRebalancerConfig.java      |  40 --
 .../config/SemiAutoRebalancerConfig.java        | 183 -------
 .../stages/BestPossibleStateCalcStage.java      |  70 +--
 .../stages/ContainerProvisioningStage.java      |  88 ++--
 .../stages/ExternalViewComputeStage.java        |   8 +-
 .../stages/MessageGenerationStage.java          |  17 +-
 .../stages/MessageSelectionStage.java           |  21 +-
 .../stages/ResourceComputationStage.java        |  51 +-
 .../java/org/apache/helix/model/IdealState.java |  83 +--
 .../helix/model/builder/IdealStateBuilder.java  |  10 +-
 .../monitoring/mbeans/ClusterStatusMonitor.java |   4 +-
 .../java/org/apache/helix/task/JobContext.java  |   6 +-
 .../java/org/apache/helix/task/TaskDriver.java  |  61 +--
 .../org/apache/helix/task/TaskRebalancer.java   |  21 +-
 .../java/org/apache/helix/task/TaskState.java   |   5 +
 .../org/apache/helix/api/TestNewStages.java     | 253 ---------
 .../context/TestSerializeRebalancerContext.java | 105 ----
 .../helix/controller/stages/BaseStageTest.java  |   5 +-
 .../stages/TestResourceComputationStage.java    |  14 +-
 .../stages/TestStagesWithLogicalAccessors.java  | 255 +++++++++
 .../TestCustomizedIdealStateRebalancer.java     |   6 +-
 .../helix/integration/TestHelixConnection.java  |  20 +-
 .../integration/TestLocalContainerProvider.java |  20 +-
 .../TestUserDefRebalancerCompatibility.java     |   3 +-
 .../mbeans/TestClusterStatusMonitor.java        |  20 +-
 .../helix/examples/LogicalModelExample.java     |  66 +--
 .../tools/UpdateProvisionerConfig.java          |   1 +
 .../provisioning/yarn/AppMasterLauncher.java    |  22 +-
 .../provisioning/yarn/YarnProvisioner.java      |   6 +-
 36 files changed, 522 insertions(+), 2109 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/helix/blob/59391888/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/helix/blob/59391888/helix-core/src/test/java/org/apache/helix/controller/stages/TestStagesWithLogicalAccessors.java
----------------------------------------------------------------------
diff --cc helix-core/src/test/java/org/apache/helix/controller/stages/TestStagesWithLogicalAccessors.java
index 0000000,ccd8c67..ad89985
mode 000000,100644..100644
--- a/helix-core/src/test/java/org/apache/helix/controller/stages/TestStagesWithLogicalAccessors.java
+++ b/helix-core/src/test/java/org/apache/helix/controller/stages/TestStagesWithLogicalAccessors.java
@@@ -1,0 -1,255 +1,255 @@@
+ package org.apache.helix.controller.stages;
+ 
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *   http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing,
+  * software distributed under the License is distributed on an
+  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  * KIND, either express or implied.  See the License for the
+  * specific language governing permissions and limitations
+  * under the License.
+  */
+ 
+ import java.util.Date;
+ import java.util.List;
+ import java.util.Map;
+ 
+ import org.apache.helix.HelixDataAccessor;
+ import org.apache.helix.TestHelper;
+ import org.apache.helix.api.Cluster;
+ import org.apache.helix.api.Participant;
+ import org.apache.helix.api.Resource;
+ import org.apache.helix.api.State;
+ import org.apache.helix.api.accessor.ClusterAccessor;
+ import org.apache.helix.api.config.ResourceConfig;
+ import org.apache.helix.api.id.ClusterId;
+ import org.apache.helix.api.id.ParticipantId;
+ import org.apache.helix.api.id.PartitionId;
+ import org.apache.helix.api.id.ResourceId;
+ import org.apache.helix.controller.stages.AttributeName;
+ import org.apache.helix.controller.stages.BestPossibleStateCalcStage;
+ import org.apache.helix.controller.stages.BestPossibleStateOutput;
+ import org.apache.helix.controller.stages.ClusterEvent;
+ import org.apache.helix.controller.stages.ResourceCurrentState;
 -import org.apache.helix.integration.manager.ClusterControllerManager;
 -import org.apache.helix.integration.manager.MockParticipantManager;
++import org.apache.helix.manager.zk.MockParticipant;
++import org.apache.helix.manager.zk.MockController;
+ import org.apache.helix.manager.zk.ZKHelixDataAccessor;
+ import org.apache.helix.model.CurrentState;
+ import org.apache.helix.model.IdealState;
+ import org.apache.helix.model.IdealState.RebalanceMode;
+ import org.apache.helix.model.ResourceAssignment;
+ import org.apache.helix.testutil.ZkTestBase;
+ import org.apache.helix.tools.ClusterStateVerifier;
+ import org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier;
+ import org.testng.Assert;
+ import org.testng.annotations.AfterClass;
+ import org.testng.annotations.BeforeClass;
+ import org.testng.annotations.Test;
+ 
+ import com.google.common.base.Function;
+ import com.google.common.collect.Maps;
+ 
+ public class TestStagesWithLogicalAccessors extends ZkTestBase {
+   final int n = 2;
+   final int p = 8;
+   final int r = 2;
 -  MockParticipantManager[] _participants = new MockParticipantManager[n];
 -  ClusterControllerManager _controller;
++  MockParticipant[] _participants = new MockParticipant[n];
++  MockController _controller;
+ 
+   ClusterId _clusterId;
+   HelixDataAccessor _dataAccessor;
+ 
+   @Test
+   public void testReadClusterDataStage() {
+     String className = TestHelper.getTestClassName();
+     String methodName = TestHelper.getTestMethodName();
+     String testName = className + "_" + methodName;
+ 
+     System.out.println("START " + testName + " at " + new Date(System.currentTimeMillis()));
+ 
+     ClusterAccessor clusterAccessor = new ClusterAccessor(_clusterId, _dataAccessor);
+     Cluster cluster = clusterAccessor.readCluster();
+ 
+     ClusterId id = cluster.getId();
+     Assert.assertEquals(id, _clusterId);
+     Map<ParticipantId, Participant> liveParticipantMap = cluster.getLiveParticipantMap();
+     Assert.assertEquals(liveParticipantMap.size(), n);
+ 
+     for (ParticipantId participantId : liveParticipantMap.keySet()) {
+       Participant participant = liveParticipantMap.get(participantId);
+       Map<ResourceId, CurrentState> curStateMap = participant.getCurrentStateMap();
+       Assert.assertEquals(curStateMap.size(), 1);
+ 
+       ResourceId resourceId = ResourceId.from("TestDB0");
+       Assert.assertTrue(curStateMap.containsKey(resourceId));
+       CurrentState curState = curStateMap.get(resourceId);
+       Map<PartitionId, State> partitionStateMap = curState.getTypedPartitionStateMap();
+       Assert.assertEquals(partitionStateMap.size(), p);
+     }
+ 
+     Map<ResourceId, Resource> resourceMap = cluster.getResourceMap();
+     Assert.assertEquals(resourceMap.size(), 1);
+ 
+     ResourceId resourceId = ResourceId.from("TestDB0");
+     Assert.assertTrue(resourceMap.containsKey(resourceId));
+     Resource resource = resourceMap.get(resourceId);
+     Assert.assertNotNull(resource.getIdealState());
+     Assert.assertEquals(resource.getIdealState().getRebalanceMode(), RebalanceMode.SEMI_AUTO);
+ 
+     System.out.println("END " + testName + " at " + new Date(System.currentTimeMillis()));
+   }
+ 
+   @Test
+   public void testBasicBestPossibleStateCalcStage() {
+     String className = TestHelper.getTestClassName();
+     String methodName = TestHelper.getTestMethodName();
+     String testName = className + "_" + methodName;
+ 
+     System.out.println("START " + testName + " at " + new Date(System.currentTimeMillis()));
+ 
+     // Set up the event
+     ClusterAccessor clusterAccessor = new ClusterAccessor(_clusterId, _dataAccessor);
+     Cluster cluster = clusterAccessor.readCluster();
+     ClusterEvent event = new ClusterEvent(testName);
+     event.addAttribute(AttributeName.CURRENT_STATE.toString(), new ResourceCurrentState());
+     Map<ResourceId, ResourceConfig> resourceConfigMap =
+         Maps.transformValues(cluster.getResourceMap(), new Function<Resource, ResourceConfig>()
{
+           @Override
+           public ResourceConfig apply(Resource resource) {
+             return resource.getConfig();
+           }
+         });
+     event.addAttribute(AttributeName.RESOURCES.toString(), resourceConfigMap);
+     event.addAttribute("Cluster", cluster);
+ 
+     // Run the stage
+     try {
+       new BestPossibleStateCalcStage().process(event);
+     } catch (Exception e) {
+       Assert.fail(e.toString());
+     }
+ 
+     // Verify the result
+     BestPossibleStateOutput bestPossibleStateOutput =
+         event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.toString());
+     Assert.assertNotNull(bestPossibleStateOutput);
+     ResourceId resourceId = ResourceId.from("TestDB0");
+     ResourceAssignment assignment = bestPossibleStateOutput.getResourceAssignment(resourceId);
+     Assert.assertNotNull(assignment);
+     Resource resource = cluster.getResource(resourceId);
+     verifySemiAutoRebalance(resource, assignment);
+ 
+     System.out.println("END " + testName + " at " + new Date(System.currentTimeMillis()));
+   }
+ 
+   @Test
+   public void testClusterRebalancers() {
+     String className = TestHelper.getTestClassName();
+     String methodName = TestHelper.getTestMethodName();
+     String testName = className + "_" + methodName;
+ 
+     System.out.println("START " + testName + " at " + new Date(System.currentTimeMillis()));
+ 
+     ClusterAccessor clusterAccessor = new ClusterAccessor(_clusterId, _dataAccessor);
+     Cluster cluster = clusterAccessor.readCluster();
+ 
+     ResourceId resourceId = ResourceId.from("TestDB0");
+     Resource resource = cluster.getResource(resourceId);
+     ResourceCurrentState currentStateOutput = new ResourceCurrentState();
+     ResourceAssignment semiAutoResult =
+         resource
+             .getIdealState()
+             .getRebalancerRef()
+             .getRebalancer()
+             .computeResourceMapping(resource.getIdealState(), resource.getRebalancerConfig(),
null,
+                 cluster, currentStateOutput);
+     verifySemiAutoRebalance(resource, semiAutoResult);
+ 
+     System.out.println("END " + testName + " at " + new Date(System.currentTimeMillis()));
+   }
+ 
+   /**
+    * Check that a semi auto rebalance is run, and all partitions are mapped by preference
list
+    * @param resource the resource to verify
+    * @param assignment the assignment to verify
+    */
+   private void verifySemiAutoRebalance(Resource resource, ResourceAssignment assignment)
{
+     Assert.assertEquals(assignment.getMappedPartitionIds().size(), resource.getSubUnitSet().size());
+     IdealState idealState = resource.getIdealState();
+     for (PartitionId partitionId : assignment.getMappedPartitionIds()) {
+       List<ParticipantId> preferenceList = idealState.getPreferenceList(partitionId);
+       Map<ParticipantId, State> replicaMap = assignment.getReplicaMap(partitionId);
+       Assert.assertEquals(replicaMap.size(), preferenceList.size());
+       Assert.assertEquals(replicaMap.size(), r);
+       boolean hasMaster = false;
+       for (ParticipantId participant : preferenceList) {
+         Assert.assertTrue(replicaMap.containsKey(participant));
+         State state = replicaMap.get(participant);
+         if (state.equals(State.from("MASTER"))) {
+           Assert.assertFalse(hasMaster);
+           hasMaster = true;
+         }
+       }
+       Assert.assertEquals(replicaMap.get(preferenceList.get(0)), State.from("MASTER"));
+     }
+   }
+ 
+   @BeforeClass
+   public void beforeClass() throws Exception {
+     // set up a running class
+     // Logger.getRootLogger().setLevel(Level.INFO);
+     String className = TestHelper.getTestClassName();
+     String methodName = TestHelper.getTestMethodName();
+     String clusterName = className + "_" + methodName;
+     _clusterId = ClusterId.from(clusterName);
+ 
+     System.out.println("START " + _clusterId + " at " + new Date(System.currentTimeMillis()));
+ 
+     TestHelper.setupCluster(clusterName, _zkaddr, 12918, // participant port
+         "localhost", // participant name prefix
+         "TestDB", // resource name prefix
+         1, // resources
+         p, // partitions per resource
+         n, // number of nodes
+         r, // replicas
+         "MasterSlave", true); // do rebalance
+ 
 -    _controller = new ClusterControllerManager(_zkaddr, clusterName, "controller_0");
++    _controller = new MockController(_zkaddr, clusterName, "controller_0");
+     _controller.syncStart();
+ 
+     // start participants
+     for (int i = 0; i < n; i++) {
+       String instanceName = "localhost_" + (12918 + i);
+ 
 -      _participants[i] = new MockParticipantManager(_zkaddr, clusterName, instanceName);
++      _participants[i] = new MockParticipant(_zkaddr, clusterName, instanceName);
+       _participants[i].syncStart();
+     }
+ 
+     boolean result =
+         ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(_zkaddr,
+             clusterName));
+     Assert.assertTrue(result);
+ 
+     _dataAccessor = new ZKHelixDataAccessor(clusterName, _baseAccessor);
+   }
+ 
+   @AfterClass
+   public void afterClass() {
+     // tear down the cluster
+     _controller.syncStop();
+     for (int i = 0; i < n; i++) {
+       _participants[i].syncStop();
+     }
+ 
+     System.out.println("END " + _clusterId + " at " + new Date(System.currentTimeMillis()));
+   }
+ }

http://git-wip-us.apache.org/repos/asf/helix/blob/59391888/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/AppMasterLauncher.java
----------------------------------------------------------------------
diff --cc helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/AppMasterLauncher.java
index 9d56cdf,6d96849..9a527a2
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/AppMasterLauncher.java
+++ b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/AppMasterLauncher.java
@@@ -45,13 -45,12 +45,12 @@@ import org.apache.helix.api.config.Clus
  import org.apache.helix.api.config.ResourceConfig;
  import org.apache.helix.api.id.ClusterId;
  import org.apache.helix.api.id.ControllerId;
+ import org.apache.helix.api.id.PartitionId;
  import org.apache.helix.api.id.ResourceId;
- import org.apache.helix.controller.rebalancer.config.FullAutoRebalancerConfig;
- import org.apache.helix.controller.rebalancer.config.PartitionedRebalancerConfig;
- import org.apache.helix.controller.rebalancer.config.RebalancerConfig;
 -import org.apache.helix.manager.zk.HelixConnectionAdaptor;
 +import org.apache.helix.manager.zk.ZKHelixManager;
  import org.apache.helix.manager.zk.ZkHelixConnection;
  import org.apache.helix.model.StateModelDefinition;
+ import org.apache.helix.model.builder.AutoRebalanceModeISBuilder;
  import org.apache.helix.provisioning.ApplicationSpec;
  import org.apache.helix.provisioning.ApplicationSpecFactory;
  import org.apache.helix.provisioning.HelixYarnUtil;


Mime
View raw message