Return-Path: X-Original-To: apmail-helix-commits-archive@minotaur.apache.org Delivered-To: apmail-helix-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 71BA610661 for ; Mon, 26 Aug 2013 21:57:10 +0000 (UTC) Received: (qmail 17117 invoked by uid 500); 26 Aug 2013 21:57:09 -0000 Delivered-To: apmail-helix-commits-archive@helix.apache.org Received: (qmail 17095 invoked by uid 500); 26 Aug 2013 21:57:09 -0000 Mailing-List: contact commits-help@helix.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@helix.incubator.apache.org Delivered-To: mailing list commits@helix.incubator.apache.org Received: (qmail 17088 invoked by uid 99); 26 Aug 2013 21:57:09 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 26 Aug 2013 21:57:09 +0000 X-ASF-Spam-Status: No, hits=-2002.2 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Mon, 26 Aug 2013 21:56:56 +0000 Received: (qmail 16996 invoked by uid 99); 26 Aug 2013 21:56:33 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 26 Aug 2013 21:56:33 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id B4C0F8C5E42; Mon, 26 Aug 2013 21:56:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zzhang@apache.org To: commits@helix.incubator.apache.org Date: Mon, 26 Aug 2013 21:56:32 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/3] [HELIX-174] Clean up ideal state calculators, move them to the controller rebalancer package, rb=13696 X-Virus-Checked: Checked by ClamAV on apache.org Updated Branches: refs/heads/master f73b3a7ae -> 21c4fcb51 http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/main/java/org/apache/helix/tools/YAISCalculator.java ---------------------------------------------------------------------- diff --git a/helix-core/src/main/java/org/apache/helix/tools/YAISCalculator.java b/helix-core/src/main/java/org/apache/helix/tools/YAISCalculator.java deleted file mode 100644 index 4292baa..0000000 --- a/helix-core/src/main/java/org/apache/helix/tools/YAISCalculator.java +++ /dev/null @@ -1,174 +0,0 @@ -package org.apache.helix.tools; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Random; - -public class YAISCalculator { - static class Assignment { - private final int numNodes; - private final int replication; - Partition[] partitions; - Node[] nodes; - - public Assignment(int numNodes, int numPartitions, int replication) { - this.numNodes = numNodes; - this.replication = replication; - partitions = new Partition[numPartitions]; - for (int i = 0; i < numPartitions; i++) { - partitions[i] = new Partition(i, replication); - } - nodes = new Node[numNodes]; - for (int i = 0; i < numNodes; i++) { - nodes[i] = new Node(replication); - } - } - - public void assign(int partitionId, int replicaId, int nodeId) { - System.out.println("Assigning (" + partitionId + "," + replicaId + ") to " + nodeId); - partitions[partitionId].nodeIds[replicaId] = nodeId; - nodes[nodeId].partitionLists.get(replicaId).push(partitionId); - } - - public void unassign(int partitionId, int replicaId) { - - } - - Integer[] getPartitionsPerNode(int nodeId, int replicaId) { - List partitionsList = new ArrayList(); - for (Partition p : partitions) { - if (p.nodeIds[replicaId] == nodeId) { - partitionsList.add(p.partionId); - } - } - Integer[] array = new Integer[partitionsList.size()]; - partitionsList.toArray(array); - return array; - } - - public void printPerNode() { - for (int nodeId = 0; nodeId < numNodes; nodeId++) { - for (int r = 0; r < replication; r++) { - StringBuilder sb = new StringBuilder(); - sb.append("(").append(nodeId).append(",").append(r).append("):\t"); - Node node = nodes[nodeId]; - LinkedList linkedList = node.partitionLists.get(r); - for (int partitionId : linkedList) { - sb.append(partitionId).append(","); - } - System.out.println(sb.toString()); - } - - } - } - } - - static class Partition { - - final int partionId; - - public Partition(int partionId, int replication) { - this.partionId = partionId; - nodeIds = new int[replication]; - Arrays.fill(nodeIds, -1); - } - - int nodeIds[]; - } - - static class Node { - private final int replication; - ArrayList> partitionLists; - - public Node(int replication) { - this.replication = replication; - partitionLists = new ArrayList>(replication); - for (int i = 0; i < replication; i++) { - partitionLists.add(new LinkedList()); - } - } - - } - - public static void main(String[] args) { - doAssignment(new int[] { - 5 - }, 120, 3); - } - - private static void doAssignment(int[] nodes, int partitions, int replication) { - int N = nodes[0]; - int totalNodes = 0; - for (int temp : nodes) { - totalNodes += temp; - } - Assignment assignment = new Assignment(totalNodes, partitions, replication); - int nodeId = 0; - for (int i = 0; i < partitions; i++) { - assignment.assign(i, 0, nodeId); - nodeId = (nodeId + 1) % N; - } - Random random = new Random(); - for (int r = 1; r < replication; r++) { - for (int id = 0; id < N; id++) { - Integer[] partitionsPerNode = assignment.getPartitionsPerNode(id, 0); - boolean[] used = new boolean[partitionsPerNode.length]; - Arrays.fill(used, false); - System.out.println(id + "-" + partitionsPerNode.length); - nodeId = (id + r) % N; - int count = partitionsPerNode.length; - boolean done = false; - do { - if (nodeId != id) { - int nextInt = random.nextInt(count); - int temp = 0; - for (int b = 0; b < used.length; b++) { - if (!used[b] && temp == nextInt) { - assignment.assign(partitionsPerNode[b], r, nodeId); - used[b] = true; - break; - } - } - } - nodeId = (nodeId + 1) % N; - } while (count > 0); - - } - } - if (nodes.length > 1) { - int prevNodeCount = nodes[0]; - for (int i = 1; i < nodes.length; i++) { - int newNodeCount = prevNodeCount + nodes[i]; - int masterPartitionsToMove = - (int) ((partitions * 1.0 / prevNodeCount - partitions * 1.0 / newNodeCount) * 1 * prevNodeCount); - while (masterPartitionsToMove > 0) { - - } - - } - } - assignment.printPerNode(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/main/java/org/apache/helix/util/RebalanceUtil.java ---------------------------------------------------------------------- diff --git a/helix-core/src/main/java/org/apache/helix/util/RebalanceUtil.java b/helix-core/src/main/java/org/apache/helix/util/RebalanceUtil.java index 3f8c406..273adc3 100644 --- a/helix-core/src/main/java/org/apache/helix/util/RebalanceUtil.java +++ b/helix-core/src/main/java/org/apache/helix/util/RebalanceUtil.java @@ -90,8 +90,8 @@ public class RebalanceUtil { } Map result = new TreeMap(); - result.put("MasterAssignmentMap", nodeMasterAssignmentMap); - result.put("SlaveAssignmentMap", combinedNodeSlaveAssignmentMap); + result.put("PrimaryAssignmentMap", nodeMasterAssignmentMap); + result.put("SecondaryAssignmentMap", combinedNodeSlaveAssignmentMap); result.put("replicas", Integer.parseInt(state.getReplicas())); result.put("partitions", new Integer(state.getRecord().getListFields().size())); result.put("reversePartitionIndex", reversePartitionIndex); http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/TestEspressoStorageClusterIdealState.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/TestEspressoStorageClusterIdealState.java b/helix-core/src/test/java/org/apache/helix/TestEspressoStorageClusterIdealState.java deleted file mode 100644 index 9b249ec..0000000 --- a/helix-core/src/test/java/org/apache/helix/TestEspressoStorageClusterIdealState.java +++ /dev/null @@ -1,301 +0,0 @@ -package org.apache.helix; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.apache.helix.ZNRecord; -import org.apache.helix.model.IdealState; -import org.apache.helix.tools.ClusterSetup; -import org.apache.helix.tools.DefaultIdealStateCalculator; -import org.apache.helix.util.RebalanceUtil; -import org.testng.Assert; -import org.testng.AssertJUnit; -import org.testng.annotations.Test; - -public class TestEspressoStorageClusterIdealState { - @Test() - public void testEspressoStorageClusterIdealState() throws Exception { - List instanceNames = new ArrayList(); - for (int i = 0; i < 5; i++) { - instanceNames.add("localhost:123" + i); - } - int partitions = 8, replicas = 0; - Map result0 = - DefaultIdealStateCalculator.calculateInitialIdealState(instanceNames, partitions, replicas); - Verify(result0, partitions, replicas); - - partitions = 8192; - replicas = 3; - - instanceNames.clear(); - for (int i = 0; i < 20; i++) { - instanceNames.add("localhost:123" + i); - } - Map resultOriginal = - DefaultIdealStateCalculator.calculateInitialIdealState(instanceNames, partitions, replicas); - - Verify(resultOriginal, partitions, replicas); - printStat(resultOriginal); - - Map result1 = - DefaultIdealStateCalculator.calculateInitialIdealState(instanceNames, partitions, replicas); - - List instanceNames2 = new ArrayList(); - for (int i = 30; i < 35; i++) { - instanceNames2.add("localhost:123" + i); - } - - DefaultIdealStateCalculator.calculateNextIdealState(instanceNames2, result1); - - List instanceNames3 = new ArrayList(); - for (int i = 35; i < 40; i++) { - instanceNames3.add("localhost:123" + i); - } - - DefaultIdealStateCalculator.calculateNextIdealState(instanceNames3, result1); - Double masterKeepRatio = 0.0, slaveKeepRatio = 0.0; - Verify(result1, partitions, replicas); - double[] result = compareResult(resultOriginal, result1); - masterKeepRatio = result[0]; - slaveKeepRatio = result[1]; - Assert.assertTrue(0.66 < masterKeepRatio && 0.67 > masterKeepRatio); - Assert.assertTrue(0.66 < slaveKeepRatio && 0.67 > slaveKeepRatio); - - } - - @Test - public void testRebalance2() { - int partitions = 1256, replicas = 3; - List instanceNames = new ArrayList(); - - for (int i = 0; i < 10; i++) { - instanceNames.add("localhost:123" + i); - } - - Map resultOriginal = - DefaultIdealStateCalculator.calculateInitialIdealState(instanceNames, partitions, replicas); - - ZNRecord idealState1 = - DefaultIdealStateCalculator.convertToZNRecord(resultOriginal, "TestDB", "MASTER", "SLAVE"); - - Map result1 = - RebalanceUtil.buildInternalIdealState(new IdealState(idealState1)); - - List instanceNames2 = new ArrayList(); - for (int i = 30; i < 35; i++) { - instanceNames2.add("localhost:123" + i); - } - - Map result2 = - DefaultIdealStateCalculator.calculateNextIdealState(instanceNames2, result1); - - Verify(resultOriginal, partitions, replicas); - Verify(result2, partitions, replicas); - Double masterKeepRatio = 0.0, slaveKeepRatio = 0.0; - double[] result = compareResult(resultOriginal, result2); - masterKeepRatio = result[0]; - slaveKeepRatio = result[1]; - Assert.assertTrue(0.66 < masterKeepRatio && 0.67 > masterKeepRatio); - Assert.assertTrue(0.66 < slaveKeepRatio && 0.67 > slaveKeepRatio); - } - - public static void Verify(Map result, int partitions, int replicas) { - Map> masterAssignmentMap = - (Map>) (result.get("MasterAssignmentMap")); - Map>> nodeSlaveAssignmentMap = - (Map>>) (result.get("SlaveAssignmentMap")); - - AssertJUnit.assertTrue(partitions == (Integer) (result.get("partitions"))); - - // Verify master partitions covers all master partitions on each node - Map masterCounterMap = new TreeMap(); - for (int i = 0; i < partitions; i++) { - masterCounterMap.put(i, 0); - } - - int minMasters = Integer.MAX_VALUE, maxMasters = Integer.MIN_VALUE; - for (String instanceName : masterAssignmentMap.keySet()) { - List masterList = masterAssignmentMap.get(instanceName); - // the assert needs to be changed when weighting is introduced - // AssertJUnit.assertTrue(masterList.size() == partitions /masterAssignmentMap.size() | - // masterList.size() == (partitions /masterAssignmentMap.size()+1) ); - - for (Integer x : masterList) { - AssertJUnit.assertTrue(masterCounterMap.get(x) == 0); - masterCounterMap.put(x, 1); - } - if (minMasters > masterList.size()) { - minMasters = masterList.size(); - } - if (maxMasters < masterList.size()) { - maxMasters = masterList.size(); - } - } - // Master partition should be evenly distributed most of the time - System.out.println("Masters: max: " + maxMasters + " Min:" + minMasters); - // Each master partition should occur only once - for (int i = 0; i < partitions; i++) { - AssertJUnit.assertTrue(masterCounterMap.get(i) == 1); - } - AssertJUnit.assertTrue(masterCounterMap.size() == partitions); - - // for each node, verify the master partitions and the slave partition assignment map - if (replicas == 0) { - AssertJUnit.assertTrue(nodeSlaveAssignmentMap.size() == 0); - return; - } - - AssertJUnit.assertTrue(masterAssignmentMap.size() == nodeSlaveAssignmentMap.size()); - for (String instanceName : masterAssignmentMap.keySet()) { - AssertJUnit.assertTrue(nodeSlaveAssignmentMap.containsKey(instanceName)); - - Map> slaveAssignmentMap = nodeSlaveAssignmentMap.get(instanceName); - Map slaveCountMap = new TreeMap(); - List masterList = masterAssignmentMap.get(instanceName); - - for (Integer masterPartitionId : masterList) { - slaveCountMap.put(masterPartitionId, 0); - } - // Make sure that masterList are covered replica times by the slave assignment. - int minSlaves = Integer.MAX_VALUE, maxSlaves = Integer.MIN_VALUE; - for (String hostInstance : slaveAssignmentMap.keySet()) { - List slaveAssignment = slaveAssignmentMap.get(hostInstance); - Set occurenceSet = new HashSet(); - - // Each slave should occur only once in the list, since the list is per-node slaves - for (Integer slavePartition : slaveAssignment) { - AssertJUnit.assertTrue(!occurenceSet.contains(slavePartition)); - occurenceSet.add(slavePartition); - - slaveCountMap.put(slavePartition, slaveCountMap.get(slavePartition) + 1); - } - if (minSlaves > slaveAssignment.size()) { - minSlaves = slaveAssignment.size(); - } - if (maxSlaves < slaveAssignment.size()) { - maxSlaves = slaveAssignment.size(); - } - } - // check if slave distribution is even - AssertJUnit.assertTrue(maxSlaves - minSlaves <= 1); - // System.out.println("Slaves: max: "+maxSlaves+" Min:"+ minSlaves); - - // for each node, the slave assignment map should cover the masters for exactly replica - // times - AssertJUnit.assertTrue(slaveCountMap.size() == masterList.size()); - for (Integer masterPartitionId : masterList) { - AssertJUnit.assertTrue(slaveCountMap.get(masterPartitionId) == replicas); - } - } - - } - - public void printStat(Map result) { - // print out master distribution - - // print out slave distribution - - } - - public static double[] compareResult(Map result1, Map result2) { - double[] result = new double[2]; - Map> masterAssignmentMap1 = - (Map>) (result1.get("MasterAssignmentMap")); - Map>> nodeSlaveAssignmentMap1 = - (Map>>) (result1.get("SlaveAssignmentMap")); - - Map> masterAssignmentMap2 = - (Map>) (result2.get("MasterAssignmentMap")); - Map>> nodeSlaveAssignmentMap2 = - (Map>>) (result2.get("SlaveAssignmentMap")); - - int commonMasters = 0; - int commonSlaves = 0; - int partitions = (Integer) (result1.get("partitions")); - int replicas = (Integer) (result1.get("replicas")); - - AssertJUnit.assertTrue((Integer) (result2.get("partitions")) == partitions); - AssertJUnit.assertTrue((Integer) (result2.get("replicas")) == replicas); - - // masterMap1 maps from partition id to the holder instance name - Map masterMap1 = new TreeMap(); - for (String instanceName : masterAssignmentMap1.keySet()) { - List masterList1 = masterAssignmentMap1.get(instanceName); - for (Integer partition : masterList1) { - AssertJUnit.assertTrue(!masterMap1.containsKey(partition)); - masterMap1.put(partition, instanceName); - } - } - // go through masterAssignmentMap2 and find out the common number - for (String instanceName : masterAssignmentMap2.keySet()) { - List masterList2 = masterAssignmentMap2.get(instanceName); - for (Integer partition : masterList2) { - if (masterMap1.get(partition).equalsIgnoreCase(instanceName)) { - commonMasters++; - } - } - } - - result[0] = 1.0 * commonMasters / partitions; - System.out.println(commonMasters + " master partitions are kept, " - + (partitions - commonMasters) + " moved, keep ratio:" + 1.0 * commonMasters / partitions); - - // maps from the partition id to the instance names that holds its slave partition - Map> slaveMap1 = new TreeMap>(); - for (String instanceName : nodeSlaveAssignmentMap1.keySet()) { - Map> slaveAssignment1 = nodeSlaveAssignmentMap1.get(instanceName); - for (String slaveHostName : slaveAssignment1.keySet()) { - List slaveList = slaveAssignment1.get(slaveHostName); - for (Integer partition : slaveList) { - if (!slaveMap1.containsKey(partition)) { - slaveMap1.put(partition, new TreeSet()); - } - AssertJUnit.assertTrue(!slaveMap1.get(partition).contains(slaveHostName)); - slaveMap1.get(partition).add(slaveHostName); - } - } - } - - for (String instanceName : nodeSlaveAssignmentMap2.keySet()) { - Map> slaveAssignment2 = nodeSlaveAssignmentMap2.get(instanceName); - for (String slaveHostName : slaveAssignment2.keySet()) { - List slaveList = slaveAssignment2.get(slaveHostName); - for (Integer partition : slaveList) { - if (slaveMap1.get(partition).contains(slaveHostName)) { - commonSlaves++; - } - } - } - } - result[1] = 1.0 * commonSlaves / partitions / replicas; - System.out.println(commonSlaves + " slave partitions are kept, " - + (partitions * replicas - commonSlaves) + " moved. keep ratio:" + 1.0 * commonSlaves - / partitions / replicas); - return result; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/TestRelayIdealStateCalculator.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/TestRelayIdealStateCalculator.java b/helix-core/src/test/java/org/apache/helix/TestRelayIdealStateCalculator.java deleted file mode 100644 index ca58659..0000000 --- a/helix-core/src/test/java/org/apache/helix/TestRelayIdealStateCalculator.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.apache.helix; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -import org.apache.helix.model.IdealState; -import org.apache.helix.tools.IdealStateCalculatorForEspressoRelay; -import org.testng.Assert; -import org.testng.annotations.Test; - -public class TestRelayIdealStateCalculator { - @Test() - public void testEspressoStorageClusterIdealState() throws Exception { - testEspressoStorageClusterIdealState(15, 9, 3); - testEspressoStorageClusterIdealState(15, 6, 3); - testEspressoStorageClusterIdealState(15, 6, 2); - testEspressoStorageClusterIdealState(6, 4, 2); - } - - public void testEspressoStorageClusterIdealState(int partitions, int nodes, int replica) - throws Exception { - List storageNodes = new ArrayList(); - for (int i = 0; i < partitions; i++) { - storageNodes.add("localhost:123" + i); - } - - List relays = new ArrayList(); - for (int i = 0; i < nodes; i++) { - relays.add("relay:123" + i); - } - - IdealState idealstate = - IdealStateCalculatorForEspressoRelay.calculateRelayIdealState(storageNodes, relays, "TEST", - replica, "Leader", "Standby", "LeaderStandby"); - - Assert.assertEquals(idealstate.getRecord().getListFields().size(), idealstate.getRecord() - .getMapFields().size()); - - Map countMap = new TreeMap(); - for (String key : idealstate.getRecord().getListFields().keySet()) { - Assert.assertEquals(idealstate.getRecord().getListFields().get(key).size(), idealstate - .getRecord().getMapFields().get(key).size()); - List list = idealstate.getRecord().getListFields().get(key); - Map map = idealstate.getRecord().getMapFields().get(key); - Assert.assertEquals(list.size(), replica); - for (String val : list) { - if (!countMap.containsKey(val)) { - countMap.put(val, 1); - } else { - countMap.put(val, countMap.get(val) + 1); - } - Assert.assertTrue(map.containsKey(val)); - } - } - for (String nodeName : countMap.keySet()) { - Assert.assertTrue(countMap.get(nodeName) <= partitions * replica / nodes + 1); - // System.out.println(nodeName + " " + countMap.get(nodeName)); - } - System.out.println(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/TestShuffledIdealState.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/TestShuffledIdealState.java b/helix-core/src/test/java/org/apache/helix/TestShuffledIdealState.java deleted file mode 100644 index 5c158f8..0000000 --- a/helix-core/src/test/java/org/apache/helix/TestShuffledIdealState.java +++ /dev/null @@ -1,253 +0,0 @@ -package org.apache.helix; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.io.IOException; -import java.io.StringReader; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.helix.ZNRecord; -import org.apache.helix.tools.IdealCalculatorByConsistentHashing; -import org.apache.helix.tools.IdealStateCalculatorByRush; -import org.apache.helix.tools.IdealStateCalculatorByShuffling; -import org.codehaus.jackson.JsonGenerationException; -import org.codehaus.jackson.map.JsonMappingException; -import org.codehaus.jackson.map.ObjectMapper; -import org.testng.Assert; -import org.testng.AssertJUnit; -import org.testng.annotations.Test; - -public class TestShuffledIdealState { - @Test() - public void testInvocation() throws Exception { - int partitions = 6, replicas = 2; - String dbName = "espressoDB1"; - List instanceNames = new ArrayList(); - instanceNames.add("localhost_1231"); - instanceNames.add("localhost_1232"); - instanceNames.add("localhost_1233"); - instanceNames.add("localhost_1234"); - - ZNRecord result = - IdealStateCalculatorByShuffling.calculateIdealState(instanceNames, partitions, replicas, - dbName); - IdealCalculatorByConsistentHashing.printIdealStateStats(result, "MASTER"); - IdealCalculatorByConsistentHashing.printIdealStateStats(result, "SLAVE"); - - ZNRecord result2 = - IdealStateCalculatorByRush.calculateIdealState(instanceNames, 1, partitions, replicas, - dbName); - - ZNRecord result3 = - IdealCalculatorByConsistentHashing.calculateIdealState(instanceNames, partitions, replicas, - dbName, new IdealCalculatorByConsistentHashing.FnvHash()); - IdealCalculatorByConsistentHashing.printIdealStateStats(result3, "MASTER"); - IdealCalculatorByConsistentHashing.printIdealStateStats(result3, "SLAVE"); - IdealCalculatorByConsistentHashing.printIdealStateStats(result3, ""); - IdealCalculatorByConsistentHashing.printNodeOfflineOverhead(result3); - - // System.out.println(result); - ObjectMapper mapper = new ObjectMapper(); - - // ByteArrayOutputStream baos = new ByteArrayOutputStream(); - StringWriter sw = new StringWriter(); - try { - mapper.writeValue(sw, result); - // System.out.println(sw.toString()); - - ZNRecord zn = mapper.readValue(new StringReader(sw.toString()), ZNRecord.class); - System.out.println(result.toString()); - System.out.println(zn.toString()); - AssertJUnit.assertTrue(zn.toString().equalsIgnoreCase(result.toString())); - System.out.println(); - - sw = new StringWriter(); - mapper.writeValue(sw, result2); - - ZNRecord zn2 = mapper.readValue(new StringReader(sw.toString()), ZNRecord.class); - System.out.println(result2.toString()); - System.out.println(zn2.toString()); - AssertJUnit.assertTrue(zn2.toString().equalsIgnoreCase(result2.toString())); - - sw = new StringWriter(); - mapper.writeValue(sw, result3); - System.out.println(); - - ZNRecord zn3 = mapper.readValue(new StringReader(sw.toString()), ZNRecord.class); - System.out.println(result3.toString()); - System.out.println(zn3.toString()); - AssertJUnit.assertTrue(zn3.toString().equalsIgnoreCase(result3.toString())); - System.out.println(); - - } catch (JsonGenerationException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (JsonMappingException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - @Test - public void testShuffledIdealState() { - // partitions is larger than nodes - int partitions = 6, replicas = 2, instances = 4; - String dbName = "espressoDB1"; - List instanceNames = new ArrayList(); - instanceNames.add("localhost_1231"); - instanceNames.add("localhost_1232"); - instanceNames.add("localhost_1233"); - instanceNames.add("localhost_1234"); - - ZNRecord result = - IdealStateCalculatorByShuffling.calculateIdealState(instanceNames, partitions, replicas, - dbName); - IdealCalculatorByConsistentHashing.printIdealStateStats(result, "MASTER"); - IdealCalculatorByConsistentHashing.printIdealStateStats(result, "SLAVE"); - Assert.assertTrue(verify(result)); - - // partition is less than nodes - instanceNames.clear(); - partitions = 4; - replicas = 3; - instances = 7; - - for (int i = 0; i < instances; i++) { - instanceNames.add("localhost_" + (1231 + i)); - } - result = - IdealStateCalculatorByShuffling.calculateIdealState(instanceNames, partitions, replicas, - dbName); - IdealCalculatorByConsistentHashing.printIdealStateStats(result, "MASTER"); - IdealCalculatorByConsistentHashing.printIdealStateStats(result, "SLAVE"); - Assert.assertTrue(verify(result)); - - // partitions is multiple of nodes - instanceNames.clear(); - partitions = 14; - replicas = 3; - instances = 7; - - for (int i = 0; i < instances; i++) { - instanceNames.add("localhost_" + (1231 + i)); - } - result = - IdealStateCalculatorByShuffling.calculateIdealState(instanceNames, partitions, replicas, - dbName); - IdealCalculatorByConsistentHashing.printIdealStateStats(result, "MASTER"); - IdealCalculatorByConsistentHashing.printIdealStateStats(result, "SLAVE"); - Assert.assertTrue(verify(result)); - - // nodes are multiple of partitions - instanceNames.clear(); - partitions = 4; - replicas = 3; - instances = 8; - - for (int i = 0; i < instances; i++) { - instanceNames.add("localhost_" + (1231 + i)); - } - result = - IdealStateCalculatorByShuffling.calculateIdealState(instanceNames, partitions, replicas, - dbName); - IdealCalculatorByConsistentHashing.printIdealStateStats(result, "MASTER"); - IdealCalculatorByConsistentHashing.printIdealStateStats(result, "SLAVE"); - Assert.assertTrue(verify(result)); - - // nodes are multiple of partitions - instanceNames.clear(); - partitions = 4; - replicas = 3; - instances = 12; - - for (int i = 0; i < instances; i++) { - instanceNames.add("localhost_" + (1231 + i)); - } - result = - IdealStateCalculatorByShuffling.calculateIdealState(instanceNames, partitions, replicas, - dbName); - IdealCalculatorByConsistentHashing.printIdealStateStats(result, "MASTER"); - IdealCalculatorByConsistentHashing.printIdealStateStats(result, "SLAVE"); - Assert.assertTrue(verify(result)); - - // Just fits - instanceNames.clear(); - partitions = 4; - replicas = 2; - instances = 12; - - for (int i = 0; i < instances; i++) { - instanceNames.add("localhost_" + (1231 + i)); - } - result = - IdealStateCalculatorByShuffling.calculateIdealState(instanceNames, partitions, replicas, - dbName); - IdealCalculatorByConsistentHashing.printIdealStateStats(result, "MASTER"); - IdealCalculatorByConsistentHashing.printIdealStateStats(result, "SLAVE"); - Assert.assertTrue(verify(result)); - } - - boolean verify(ZNRecord result) { - Map masterPartitionCounts = new HashMap(); - Map slavePartitionCounts = new HashMap(); - - for (String key : result.getMapFields().keySet()) { - Map mapField = result.getMapField(key); - int masterCount = 0; - for (String host : mapField.keySet()) { - if (mapField.get(host).equals("MASTER")) { - Assert.assertTrue(masterCount == 0); - masterCount++; - if (!masterPartitionCounts.containsKey(host)) { - masterPartitionCounts.put(host, 0); - } else { - masterPartitionCounts.put(host, masterPartitionCounts.get(host) + 1); - } - } else { - if (!slavePartitionCounts.containsKey(host)) { - slavePartitionCounts.put(host, 0); - } else { - slavePartitionCounts.put(host, slavePartitionCounts.get(host) + 1); - } - } - } - } - - List masterCounts = new ArrayList(); - List slaveCounts = new ArrayList(); - masterCounts.addAll(masterPartitionCounts.values()); - slaveCounts.addAll(slavePartitionCounts.values()); - Collections.sort(masterCounts); - Collections.sort(slaveCounts); - - Assert.assertTrue(masterCounts.get(masterCounts.size() - 1) - masterCounts.get(0) <= 1); - - Assert.assertTrue(slaveCounts.get(slaveCounts.size() - 1) - slaveCounts.get(0) <= 2); - return true; - } -} http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/controller/stages/TestCompatibilityCheckStage.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/controller/stages/TestCompatibilityCheckStage.java b/helix-core/src/test/java/org/apache/helix/controller/stages/TestCompatibilityCheckStage.java index 391d1af..0b97e20 100644 --- a/helix-core/src/test/java/org/apache/helix/controller/stages/TestCompatibilityCheckStage.java +++ b/helix-core/src/test/java/org/apache/helix/controller/stages/TestCompatibilityCheckStage.java @@ -23,15 +23,13 @@ import java.util.Arrays; import java.util.List; import org.apache.helix.Mocks; -import org.apache.helix.ZNRecord; import org.apache.helix.PropertyKey.Builder; +import org.apache.helix.ZNRecord; import org.apache.helix.controller.pipeline.StageContext; -import org.apache.helix.controller.stages.CompatibilityCheckStage; -import org.apache.helix.controller.stages.ReadClusterDataStage; +import org.apache.helix.controller.strategy.DefaultTwoStateStrategy; import org.apache.helix.model.IdealState; import org.apache.helix.model.LiveInstance; import org.apache.helix.model.LiveInstance.LiveInstanceProperty; -import org.apache.helix.tools.DefaultIdealStateCalculator; import org.testng.Assert; import org.testng.annotations.Test; @@ -50,8 +48,8 @@ public class TestCompatibilityCheckStage extends BaseStageTest { // set ideal state String resourceName = "testResource"; ZNRecord record = - DefaultIdealStateCalculator.calculateIdealState(instances, partitions, replicas, - resourceName, "MASTER", "SLAVE"); + DefaultTwoStateStrategy.calculateIdealState(instances, partitions, replicas, resourceName, + "MASTER", "SLAVE"); IdealState idealState = new IdealState(record); idealState.setStateModelDefRef("MasterSlave"); http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/controller/stages/TestResourceComputationStage.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/controller/stages/TestResourceComputationStage.java b/helix-core/src/test/java/org/apache/helix/controller/stages/TestResourceComputationStage.java index dcb955c..6febe93 100644 --- a/helix-core/src/test/java/org/apache/helix/controller/stages/TestResourceComputationStage.java +++ b/helix-core/src/test/java/org/apache/helix/controller/stages/TestResourceComputationStage.java @@ -25,19 +25,15 @@ import java.util.Map; import java.util.UUID; import org.apache.helix.HelixDataAccessor; -import org.apache.helix.ZNRecord; import org.apache.helix.PropertyKey.Builder; +import org.apache.helix.ZNRecord; import org.apache.helix.controller.pipeline.StageContext; -import org.apache.helix.controller.stages.AttributeName; -import org.apache.helix.controller.stages.ClusterEvent; -import org.apache.helix.controller.stages.ReadClusterDataStage; -import org.apache.helix.controller.stages.ResourceComputationStage; +import org.apache.helix.controller.strategy.DefaultTwoStateStrategy; import org.apache.helix.model.CurrentState; import org.apache.helix.model.IdealState; +import org.apache.helix.model.IdealState.RebalanceMode; import org.apache.helix.model.LiveInstance; import org.apache.helix.model.Resource; -import org.apache.helix.model.IdealState.RebalanceMode; -import org.apache.helix.tools.DefaultIdealStateCalculator; import org.testng.AssertJUnit; import org.testng.annotations.Test; @@ -57,8 +53,8 @@ public class TestResourceComputationStage extends BaseStageTest { int replicas = 1; String resourceName = "testResource"; ZNRecord record = - DefaultIdealStateCalculator.calculateIdealState(instances, partitions, replicas, - resourceName, "MASTER", "SLAVE"); + DefaultTwoStateStrategy.calculateIdealState(instances, partitions, replicas, resourceName, + "MASTER", "SLAVE"); IdealState idealState = new IdealState(record); idealState.setStateModelDefRef("MasterSlave"); @@ -122,7 +118,7 @@ public class TestResourceComputationStage extends BaseStageTest { int replicas = 1; String resourceName = resources[i]; ZNRecord record = - DefaultIdealStateCalculator.calculateIdealState(instances, partitions, replicas, + DefaultTwoStateStrategy.calculateIdealState(instances, partitions, replicas, resourceName, "MASTER", "SLAVE"); IdealState idealState = new IdealState(record); idealState.setStateModelDefRef("MasterSlave"); http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/controller/strategy/TestEspressoRelayStrategy.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/controller/strategy/TestEspressoRelayStrategy.java b/helix-core/src/test/java/org/apache/helix/controller/strategy/TestEspressoRelayStrategy.java new file mode 100644 index 0000000..ee0d143 --- /dev/null +++ b/helix-core/src/test/java/org/apache/helix/controller/strategy/TestEspressoRelayStrategy.java @@ -0,0 +1,81 @@ +package org.apache.helix.controller.strategy; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.helix.model.IdealState; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class TestEspressoRelayStrategy { + @Test() + public void testEspressoStorageClusterIdealState() throws Exception { + testEspressoStorageClusterIdealState(15, 9, 3); + testEspressoStorageClusterIdealState(15, 6, 3); + testEspressoStorageClusterIdealState(15, 6, 2); + testEspressoStorageClusterIdealState(6, 4, 2); + } + + public void testEspressoStorageClusterIdealState(int partitions, int nodes, int replica) + throws Exception { + List storageNodes = new ArrayList(); + for (int i = 0; i < partitions; i++) { + storageNodes.add("localhost:123" + i); + } + + List relays = new ArrayList(); + for (int i = 0; i < nodes; i++) { + relays.add("relay:123" + i); + } + + IdealState idealstate = + EspressoRelayStrategy.calculateRelayIdealState(storageNodes, relays, "TEST", replica, + "Leader", "Standby", "LeaderStandby"); + + Assert.assertEquals(idealstate.getRecord().getListFields().size(), idealstate.getRecord() + .getMapFields().size()); + + Map countMap = new TreeMap(); + for (String key : idealstate.getRecord().getListFields().keySet()) { + Assert.assertEquals(idealstate.getRecord().getListFields().get(key).size(), idealstate + .getRecord().getMapFields().get(key).size()); + List list = idealstate.getRecord().getListFields().get(key); + Map map = idealstate.getRecord().getMapFields().get(key); + Assert.assertEquals(list.size(), replica); + for (String val : list) { + if (!countMap.containsKey(val)) { + countMap.put(val, 1); + } else { + countMap.put(val, countMap.get(val) + 1); + } + Assert.assertTrue(map.containsKey(val)); + } + } + for (String nodeName : countMap.keySet()) { + Assert.assertTrue(countMap.get(nodeName) <= partitions * replica / nodes + 1); + // System.out.println(nodeName + " " + countMap.get(nodeName)); + } + System.out.println(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/controller/strategy/TestEspressoStorageClusterIdealState.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/controller/strategy/TestEspressoStorageClusterIdealState.java b/helix-core/src/test/java/org/apache/helix/controller/strategy/TestEspressoStorageClusterIdealState.java new file mode 100644 index 0000000..dbb870e --- /dev/null +++ b/helix-core/src/test/java/org/apache/helix/controller/strategy/TestEspressoStorageClusterIdealState.java @@ -0,0 +1,298 @@ +package org.apache.helix.controller.strategy; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.helix.ZNRecord; +import org.apache.helix.model.IdealState; +import org.apache.helix.util.RebalanceUtil; +import org.testng.Assert; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + +public class TestEspressoStorageClusterIdealState { + @Test + public void testEspressoStorageClusterIdealState() throws Exception { + List instanceNames = new ArrayList(); + for (int i = 0; i < 5; i++) { + instanceNames.add("localhost:123" + i); + } + int partitions = 8, replicas = 0; + Map result0 = + DefaultTwoStateStrategy.calculateInitialIdealState(instanceNames, partitions, replicas); + Verify(result0, partitions, replicas); + + partitions = 8192; + replicas = 3; + + instanceNames.clear(); + for (int i = 0; i < 20; i++) { + instanceNames.add("localhost:123" + i); + } + Map resultOriginal = + DefaultTwoStateStrategy.calculateInitialIdealState(instanceNames, partitions, replicas); + + Verify(resultOriginal, partitions, replicas); + printStat(resultOriginal); + + Map result1 = + DefaultTwoStateStrategy.calculateInitialIdealState(instanceNames, partitions, replicas); + + List instanceNames2 = new ArrayList(); + for (int i = 30; i < 35; i++) { + instanceNames2.add("localhost:123" + i); + } + + DefaultTwoStateStrategy.calculateNextIdealState(instanceNames2, result1); + + List instanceNames3 = new ArrayList(); + for (int i = 35; i < 40; i++) { + instanceNames3.add("localhost:123" + i); + } + + DefaultTwoStateStrategy.calculateNextIdealState(instanceNames3, result1); + Double masterKeepRatio = 0.0, slaveKeepRatio = 0.0; + Verify(result1, partitions, replicas); + double[] result = compareResult(resultOriginal, result1); + masterKeepRatio = result[0]; + slaveKeepRatio = result[1]; + Assert.assertTrue(0.66 < masterKeepRatio && 0.67 > masterKeepRatio); + Assert.assertTrue(0.66 < slaveKeepRatio && 0.67 > slaveKeepRatio); + + } + + @Test + public void testRebalance2() { + int partitions = 1256, replicas = 3; + List instanceNames = new ArrayList(); + + for (int i = 0; i < 10; i++) { + instanceNames.add("localhost:123" + i); + } + + Map resultOriginal = + DefaultTwoStateStrategy.calculateInitialIdealState(instanceNames, partitions, replicas); + + ZNRecord idealState1 = + DefaultTwoStateStrategy.convertToZNRecord(resultOriginal, "TestDB", "MASTER", "SLAVE"); + + Map result1 = + RebalanceUtil.buildInternalIdealState(new IdealState(idealState1)); + List instanceNames2 = new ArrayList(); + for (int i = 30; i < 35; i++) { + instanceNames2.add("localhost:123" + i); + } + + Map result2 = + DefaultTwoStateStrategy.calculateNextIdealState(instanceNames2, result1); + + Verify(resultOriginal, partitions, replicas); + Verify(result2, partitions, replicas); + Double masterKeepRatio = 0.0, slaveKeepRatio = 0.0; + double[] result = compareResult(resultOriginal, result2); + masterKeepRatio = result[0]; + slaveKeepRatio = result[1]; + Assert.assertTrue(0.66 < masterKeepRatio && 0.67 > masterKeepRatio); + Assert.assertTrue(0.66 < slaveKeepRatio && 0.67 > slaveKeepRatio); + } + + public static void Verify(Map result, int partitions, int replicas) { + Map> masterAssignmentMap = + (Map>) (result.get("PrimaryAssignmentMap")); + Map>> nodeSlaveAssignmentMap = + (Map>>) (result.get("SecondaryAssignmentMap")); + + AssertJUnit.assertTrue(partitions == (Integer) (result.get("partitions"))); + + // Verify master partitions covers all master partitions on each node + Map masterCounterMap = new TreeMap(); + for (int i = 0; i < partitions; i++) { + masterCounterMap.put(i, 0); + } + + int minMasters = Integer.MAX_VALUE, maxMasters = Integer.MIN_VALUE; + for (String instanceName : masterAssignmentMap.keySet()) { + List masterList = masterAssignmentMap.get(instanceName); + // the assert needs to be changed when weighting is introduced + // AssertJUnit.assertTrue(masterList.size() == partitions /masterAssignmentMap.size() | + // masterList.size() == (partitions /masterAssignmentMap.size()+1) ); + + for (Integer x : masterList) { + AssertJUnit.assertTrue(masterCounterMap.get(x) == 0); + masterCounterMap.put(x, 1); + } + if (minMasters > masterList.size()) { + minMasters = masterList.size(); + } + if (maxMasters < masterList.size()) { + maxMasters = masterList.size(); + } + } + // Master partition should be evenly distributed most of the time + System.out.println("Masters: max: " + maxMasters + " Min:" + minMasters); + // Each master partition should occur only once + for (int i = 0; i < partitions; i++) { + AssertJUnit.assertTrue(masterCounterMap.get(i) == 1); + } + AssertJUnit.assertTrue(masterCounterMap.size() == partitions); + + // for each node, verify the master partitions and the slave partition assignment map + if (replicas == 0) { + AssertJUnit.assertTrue(nodeSlaveAssignmentMap.size() == 0); + return; + } + + AssertJUnit.assertTrue(masterAssignmentMap.size() == nodeSlaveAssignmentMap.size()); + for (String instanceName : masterAssignmentMap.keySet()) { + AssertJUnit.assertTrue(nodeSlaveAssignmentMap.containsKey(instanceName)); + + Map> slaveAssignmentMap = nodeSlaveAssignmentMap.get(instanceName); + Map slaveCountMap = new TreeMap(); + List masterList = masterAssignmentMap.get(instanceName); + + for (Integer masterPartitionId : masterList) { + slaveCountMap.put(masterPartitionId, 0); + } + // Make sure that masterList are covered replica times by the slave assignment. + int minSlaves = Integer.MAX_VALUE, maxSlaves = Integer.MIN_VALUE; + for (String hostInstance : slaveAssignmentMap.keySet()) { + List slaveAssignment = slaveAssignmentMap.get(hostInstance); + Set occurenceSet = new HashSet(); + + // Each slave should occur only once in the list, since the list is per-node slaves + for (Integer slavePartition : slaveAssignment) { + AssertJUnit.assertTrue(!occurenceSet.contains(slavePartition)); + occurenceSet.add(slavePartition); + + slaveCountMap.put(slavePartition, slaveCountMap.get(slavePartition) + 1); + } + if (minSlaves > slaveAssignment.size()) { + minSlaves = slaveAssignment.size(); + } + if (maxSlaves < slaveAssignment.size()) { + maxSlaves = slaveAssignment.size(); + } + } + // check if slave distribution is even + AssertJUnit.assertTrue(maxSlaves - minSlaves <= 1); + // System.out.println("Slaves: max: "+maxSlaves+" Min:"+ minSlaves); + + // for each node, the slave assignment map should cover the masters for exactly replica + // times + AssertJUnit.assertTrue(slaveCountMap.size() == masterList.size()); + for (Integer masterPartitionId : masterList) { + AssertJUnit.assertTrue(slaveCountMap.get(masterPartitionId) == replicas); + } + } + + } + + public void printStat(Map result) { + // print out master distribution + + // print out slave distribution + + } + + public static double[] compareResult(Map result1, Map result2) { + double[] result = new double[2]; + Map> masterAssignmentMap1 = + (Map>) (result1.get("PrimaryAssignmentMap")); + Map>> nodeSlaveAssignmentMap1 = + (Map>>) (result1.get("SecondaryAssignmentMap")); + + Map> masterAssignmentMap2 = + (Map>) (result2.get("PrimaryAssignmentMap")); + Map>> nodeSlaveAssignmentMap2 = + (Map>>) (result2.get("SecondaryAssignmentMap")); + + int commonMasters = 0; + int commonSlaves = 0; + int partitions = (Integer) (result1.get("partitions")); + int replicas = (Integer) (result1.get("replicas")); + + AssertJUnit.assertTrue((Integer) (result2.get("partitions")) == partitions); + AssertJUnit.assertTrue((Integer) (result2.get("replicas")) == replicas); + + // masterMap1 maps from partition id to the holder instance name + Map masterMap1 = new TreeMap(); + for (String instanceName : masterAssignmentMap1.keySet()) { + List masterList1 = masterAssignmentMap1.get(instanceName); + for (Integer partition : masterList1) { + AssertJUnit.assertTrue(!masterMap1.containsKey(partition)); + masterMap1.put(partition, instanceName); + } + } + // go through masterAssignmentMap2 and find out the common number + for (String instanceName : masterAssignmentMap2.keySet()) { + List masterList2 = masterAssignmentMap2.get(instanceName); + for (Integer partition : masterList2) { + if (masterMap1.get(partition).equalsIgnoreCase(instanceName)) { + commonMasters++; + } + } + } + + result[0] = 1.0 * commonMasters / partitions; + System.out.println(commonMasters + " master partitions are kept, " + + (partitions - commonMasters) + " moved, keep ratio:" + 1.0 * commonMasters / partitions); + + // maps from the partition id to the instance names that holds its slave partition + Map> slaveMap1 = new TreeMap>(); + for (String instanceName : nodeSlaveAssignmentMap1.keySet()) { + Map> slaveAssignment1 = nodeSlaveAssignmentMap1.get(instanceName); + for (String slaveHostName : slaveAssignment1.keySet()) { + List slaveList = slaveAssignment1.get(slaveHostName); + for (Integer partition : slaveList) { + if (!slaveMap1.containsKey(partition)) { + slaveMap1.put(partition, new TreeSet()); + } + AssertJUnit.assertTrue(!slaveMap1.get(partition).contains(slaveHostName)); + slaveMap1.get(partition).add(slaveHostName); + } + } + } + + for (String instanceName : nodeSlaveAssignmentMap2.keySet()) { + Map> slaveAssignment2 = nodeSlaveAssignmentMap2.get(instanceName); + for (String slaveHostName : slaveAssignment2.keySet()) { + List slaveList = slaveAssignment2.get(slaveHostName); + for (Integer partition : slaveList) { + if (slaveMap1.get(partition).contains(slaveHostName)) { + commonSlaves++; + } + } + } + } + result[1] = 1.0 * commonSlaves / partitions / replicas; + System.out.println(commonSlaves + " slave partitions are kept, " + + (partitions * replicas - commonSlaves) + " moved. keep ratio:" + 1.0 * commonSlaves + / partitions / replicas); + return result; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/controller/strategy/TestShufflingTwoStateStrategy.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/controller/strategy/TestShufflingTwoStateStrategy.java b/helix-core/src/test/java/org/apache/helix/controller/strategy/TestShufflingTwoStateStrategy.java new file mode 100644 index 0000000..0269764 --- /dev/null +++ b/helix-core/src/test/java/org/apache/helix/controller/strategy/TestShufflingTwoStateStrategy.java @@ -0,0 +1,242 @@ +package org.apache.helix.controller.strategy; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.helix.ZNRecord; +import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; +import org.testng.Assert; +import org.testng.AssertJUnit; +import org.testng.annotations.Test; + +public class TestShufflingTwoStateStrategy { + @Test() + public void testInvocation() throws Exception { + int partitions = 6, replicas = 2; + String dbName = "espressoDB1"; + List instanceNames = new ArrayList(); + instanceNames.add("localhost_1231"); + instanceNames.add("localhost_1232"); + instanceNames.add("localhost_1233"); + instanceNames.add("localhost_1234"); + + ZNRecord result = + ShufflingTwoStateStrategy.calculateIdealState(instanceNames, partitions, replicas, dbName); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result, "MASTER"); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result, "SLAVE"); + + ZNRecord result2 = + RUSHMasterSlaveStrategy.calculateIdealState(instanceNames, 1, partitions, replicas, dbName); + + ZNRecord result3 = + ConsistentHashingMasterSlaveStrategy.calculateIdealState(instanceNames, partitions, + replicas, dbName, new ConsistentHashingMasterSlaveStrategy.FnvHash()); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result3, "MASTER"); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result3, "SLAVE"); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result3, ""); + ConsistentHashingMasterSlaveStrategy.printNodeOfflineOverhead(result3); + + // System.out.println(result); + ObjectMapper mapper = new ObjectMapper(); + + // ByteArrayOutputStream baos = new ByteArrayOutputStream(); + StringWriter sw = new StringWriter(); + try { + mapper.writeValue(sw, result); + // System.out.println(sw.toString()); + + ZNRecord zn = mapper.readValue(new StringReader(sw.toString()), ZNRecord.class); + System.out.println(result.toString()); + System.out.println(zn.toString()); + AssertJUnit.assertTrue(zn.toString().equalsIgnoreCase(result.toString())); + System.out.println(); + + sw = new StringWriter(); + mapper.writeValue(sw, result2); + + ZNRecord zn2 = mapper.readValue(new StringReader(sw.toString()), ZNRecord.class); + System.out.println(result2.toString()); + System.out.println(zn2.toString()); + AssertJUnit.assertTrue(zn2.toString().equalsIgnoreCase(result2.toString())); + + sw = new StringWriter(); + mapper.writeValue(sw, result3); + System.out.println(); + + ZNRecord zn3 = mapper.readValue(new StringReader(sw.toString()), ZNRecord.class); + System.out.println(result3.toString()); + System.out.println(zn3.toString()); + AssertJUnit.assertTrue(zn3.toString().equalsIgnoreCase(result3.toString())); + System.out.println(); + + } catch (JsonGenerationException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (JsonMappingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + @Test + public void testShuffledIdealState() { + // partitions is larger than nodes + int partitions = 6, replicas = 2, instances = 4; + String dbName = "espressoDB1"; + List instanceNames = new ArrayList(); + instanceNames.add("localhost_1231"); + instanceNames.add("localhost_1232"); + instanceNames.add("localhost_1233"); + instanceNames.add("localhost_1234"); + + ZNRecord result = + ShufflingTwoStateStrategy.calculateIdealState(instanceNames, partitions, replicas, dbName); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result, "MASTER"); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result, "SLAVE"); + Assert.assertTrue(verify(result)); + + // partition is less than nodes + instanceNames.clear(); + partitions = 4; + replicas = 3; + instances = 7; + + for (int i = 0; i < instances; i++) { + instanceNames.add("localhost_" + (1231 + i)); + } + result = + ShufflingTwoStateStrategy.calculateIdealState(instanceNames, partitions, replicas, dbName); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result, "MASTER"); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result, "SLAVE"); + Assert.assertTrue(verify(result)); + + // partitions is multiple of nodes + instanceNames.clear(); + partitions = 14; + replicas = 3; + instances = 7; + + for (int i = 0; i < instances; i++) { + instanceNames.add("localhost_" + (1231 + i)); + } + result = + ShufflingTwoStateStrategy.calculateIdealState(instanceNames, partitions, replicas, dbName); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result, "MASTER"); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result, "SLAVE"); + Assert.assertTrue(verify(result)); + + // nodes are multiple of partitions + instanceNames.clear(); + partitions = 4; + replicas = 3; + instances = 8; + + for (int i = 0; i < instances; i++) { + instanceNames.add("localhost_" + (1231 + i)); + } + result = + ShufflingTwoStateStrategy.calculateIdealState(instanceNames, partitions, replicas, dbName); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result, "MASTER"); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result, "SLAVE"); + Assert.assertTrue(verify(result)); + + // nodes are multiple of partitions + instanceNames.clear(); + partitions = 4; + replicas = 3; + instances = 12; + + for (int i = 0; i < instances; i++) { + instanceNames.add("localhost_" + (1231 + i)); + } + result = + ShufflingTwoStateStrategy.calculateIdealState(instanceNames, partitions, replicas, dbName); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result, "MASTER"); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result, "SLAVE"); + Assert.assertTrue(verify(result)); + + // Just fits + instanceNames.clear(); + partitions = 4; + replicas = 2; + instances = 12; + + for (int i = 0; i < instances; i++) { + instanceNames.add("localhost_" + (1231 + i)); + } + result = + ShufflingTwoStateStrategy.calculateIdealState(instanceNames, partitions, replicas, dbName); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result, "MASTER"); + ConsistentHashingMasterSlaveStrategy.printIdealStateStats(result, "SLAVE"); + Assert.assertTrue(verify(result)); + } + + boolean verify(ZNRecord result) { + Map masterPartitionCounts = new HashMap(); + Map slavePartitionCounts = new HashMap(); + + for (String key : result.getMapFields().keySet()) { + Map mapField = result.getMapField(key); + int masterCount = 0; + for (String host : mapField.keySet()) { + if (mapField.get(host).equals("MASTER")) { + Assert.assertTrue(masterCount == 0); + masterCount++; + if (!masterPartitionCounts.containsKey(host)) { + masterPartitionCounts.put(host, 0); + } else { + masterPartitionCounts.put(host, masterPartitionCounts.get(host) + 1); + } + } else { + if (!slavePartitionCounts.containsKey(host)) { + slavePartitionCounts.put(host, 0); + } else { + slavePartitionCounts.put(host, slavePartitionCounts.get(host) + 1); + } + } + } + } + + List masterCounts = new ArrayList(); + List slaveCounts = new ArrayList(); + masterCounts.addAll(masterPartitionCounts.values()); + slaveCounts.addAll(slavePartitionCounts.values()); + Collections.sort(masterCounts); + Collections.sort(slaveCounts); + + Assert.assertTrue(masterCounts.get(masterCounts.size() - 1) - masterCounts.get(0) <= 1); + + Assert.assertTrue(slaveCounts.get(slaveCounts.size() - 1) - slaveCounts.get(0) <= 2); + return true; + } +} http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/integration/TestAutoIsWithEmptyMap.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestAutoIsWithEmptyMap.java b/helix-core/src/test/java/org/apache/helix/integration/TestAutoIsWithEmptyMap.java index 1ffb86f..a008814 100644 --- a/helix-core/src/test/java/org/apache/helix/integration/TestAutoIsWithEmptyMap.java +++ b/helix-core/src/test/java/org/apache/helix/integration/TestAutoIsWithEmptyMap.java @@ -28,10 +28,10 @@ import org.apache.helix.PropertyType; import org.apache.helix.TestHelper; import org.apache.helix.ZNRecord; import org.apache.helix.controller.HelixControllerMain; +import org.apache.helix.controller.strategy.DefaultTwoStateStrategy; import org.apache.helix.mock.participant.MockParticipant; import org.apache.helix.model.IdealState; import org.apache.helix.tools.ClusterStateVerifier; -import org.apache.helix.tools.DefaultIdealStateCalculator; import org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier; import org.testng.Assert; import org.testng.annotations.Test; @@ -63,7 +63,7 @@ public class TestAutoIsWithEmptyMap extends ZkIntegrationTestBase { instanceNames.add("localhost_" + port); } ZNRecord idealState = - DefaultIdealStateCalculator.calculateIdealState(instanceNames, 10, 2, "TestDB0", "LEADER", + DefaultTwoStateStrategy.calculateIdealState(instanceNames, 10, 2, "TestDB0", "LEADER", "STANDBY"); // System.out.println(idealState); // curIdealState.setSimpleField(IdealState.IdealStateProperty.IDEAL_STATE_MODE.toString(), http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/integration/TestDriver.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestDriver.java b/helix-core/src/test/java/org/apache/helix/integration/TestDriver.java index 951607b..b29e25d 100644 --- a/helix-core/src/test/java/org/apache/helix/integration/TestDriver.java +++ b/helix-core/src/test/java/org/apache/helix/integration/TestDriver.java @@ -32,9 +32,10 @@ import org.apache.helix.HelixManager; import org.apache.helix.PropertyPathConfig; import org.apache.helix.PropertyType; import org.apache.helix.TestHelper; -import org.apache.helix.ZNRecord; import org.apache.helix.TestHelper.StartCMResult; +import org.apache.helix.ZNRecord; import org.apache.helix.controller.HelixControllerMain; +import org.apache.helix.controller.strategy.DefaultTwoStateStrategy; import org.apache.helix.manager.zk.ZNRecordSerializer; import org.apache.helix.manager.zk.ZkClient; import org.apache.helix.model.IdealState.IdealStateProperty; @@ -43,14 +44,13 @@ import org.apache.helix.store.PropertyJsonSerializer; import org.apache.helix.store.PropertyStoreException; import org.apache.helix.tools.ClusterSetup; import org.apache.helix.tools.ClusterStateVerifier; -import org.apache.helix.tools.DefaultIdealStateCalculator; import org.apache.helix.tools.TestCommand; -import org.apache.helix.tools.TestExecutor; -import org.apache.helix.tools.TestTrigger; -import org.apache.helix.tools.ZnodeOpArg; import org.apache.helix.tools.TestCommand.CommandType; import org.apache.helix.tools.TestCommand.NodeOpArg; +import org.apache.helix.tools.TestExecutor; import org.apache.helix.tools.TestExecutor.ZnodePropertyType; +import org.apache.helix.tools.TestTrigger; +import org.apache.helix.tools.ZnodeOpArg; import org.apache.log4j.Logger; import org.testng.Assert; @@ -331,8 +331,8 @@ public class TestDriver { for (int i = 0; i < testInfo._numDb; i++) { String dbName = TEST_DB_PREFIX + i; ZNRecord destIS = - DefaultIdealStateCalculator.calculateIdealState(instanceNames, - testInfo._numPartitionsPerDb, testInfo._replica - 1, dbName, "MASTER", "SLAVE"); + DefaultTwoStateStrategy.calculateIdealState(instanceNames, testInfo._numPartitionsPerDb, + testInfo._replica - 1, dbName, "MASTER", "SLAVE"); // destIS.setId(dbName); destIS.setSimpleField(IdealStateProperty.REBALANCE_MODE.toString(), RebalanceMode.CUSTOMIZED.toString()); http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/integration/TestExpandCluster.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestExpandCluster.java b/helix-core/src/test/java/org/apache/helix/integration/TestExpandCluster.java index a9aa3b9..8268679 100644 --- a/helix-core/src/test/java/org/apache/helix/integration/TestExpandCluster.java +++ b/helix-core/src/test/java/org/apache/helix/integration/TestExpandCluster.java @@ -21,7 +21,7 @@ package org.apache.helix.integration; import java.util.Map; -import org.apache.helix.TestEspressoStorageClusterIdealState; +import org.apache.helix.controller.strategy.TestEspressoStorageClusterIdealState; import org.apache.helix.model.IdealState; import org.apache.helix.tools.ClusterSetup; import org.apache.helix.util.RebalanceUtil; http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/integration/TestRenamePartition.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestRenamePartition.java b/helix-core/src/test/java/org/apache/helix/integration/TestRenamePartition.java index ed056ab..c3133cc 100644 --- a/helix-core/src/test/java/org/apache/helix/integration/TestRenamePartition.java +++ b/helix-core/src/test/java/org/apache/helix/integration/TestRenamePartition.java @@ -24,17 +24,17 @@ import java.util.Date; import java.util.List; import java.util.Map; +import org.apache.helix.PropertyKey.Builder; import org.apache.helix.TestHelper; import org.apache.helix.ZNRecord; -import org.apache.helix.PropertyKey.Builder; import org.apache.helix.controller.HelixControllerMain; +import org.apache.helix.controller.strategy.DefaultTwoStateStrategy; import org.apache.helix.manager.zk.ZKHelixDataAccessor; import org.apache.helix.manager.zk.ZkBaseDataAccessor; import org.apache.helix.mock.participant.MockParticipant; import org.apache.helix.model.IdealState; import org.apache.helix.model.IdealState.RebalanceMode; import org.apache.helix.tools.ClusterStateVerifier; -import org.apache.helix.tools.DefaultIdealStateCalculator; import org.testng.Assert; import org.testng.annotations.Test; @@ -95,8 +95,8 @@ public class TestRenamePartition extends ZkIntegrationTestBase { Arrays.asList("localhost_12918", "localhost_12919", "localhost_12920", "localhost_12921", "localhost_12922"); ZNRecord destIS = - DefaultIdealStateCalculator.calculateIdealState(instanceNames, 10, 3 - 1, "TestDB0", - "MASTER", "SLAVE"); + DefaultTwoStateStrategy.calculateIdealState(instanceNames, 10, 3 - 1, "TestDB0", "MASTER", + "SLAVE"); IdealState idealState = new IdealState(destIS); idealState.setRebalanceMode(RebalanceMode.CUSTOMIZED); idealState.setReplicas("3"); http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/josql/TestClusterJosqlQueryProcessor.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/josql/TestClusterJosqlQueryProcessor.java b/helix-core/src/test/java/org/apache/helix/josql/TestClusterJosqlQueryProcessor.java index 8090201..30c23fb 100644 --- a/helix-core/src/test/java/org/apache/helix/josql/TestClusterJosqlQueryProcessor.java +++ b/helix-core/src/test/java/org/apache/helix/josql/TestClusterJosqlQueryProcessor.java @@ -28,10 +28,8 @@ import java.util.UUID; import org.apache.helix.Criteria; import org.apache.helix.InstanceType; import org.apache.helix.ZNRecord; -import org.apache.helix.josql.ZNRecordJosqlFunctionHandler; -import org.apache.helix.josql.ZNRecordRow; +import org.apache.helix.controller.strategy.DefaultTwoStateStrategy; import org.apache.helix.model.LiveInstance.LiveInstanceProperty; -import org.apache.helix.tools.DefaultIdealStateCalculator; import org.josql.Query; import org.josql.QueryExecutionException; import org.josql.QueryParseException; @@ -59,9 +57,7 @@ public class TestClusterJosqlQueryProcessor { // liveInstances.remove(0); ZNRecord externalView = - DefaultIdealStateCalculator.calculateIdealState(instances, 21, 3, "TestDB", "MASTER", - "SLAVE"); - + DefaultTwoStateStrategy.calculateIdealState(instances, 21, 3, "TestDB", "MASTER", "SLAVE"); Criteria criteria = new Criteria(); criteria.setInstanceName("%"); criteria.setResource("TestDB"); http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/messaging/TestDefaultMessagingService.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/messaging/TestDefaultMessagingService.java b/helix-core/src/test/java/org/apache/helix/messaging/TestDefaultMessagingService.java index 9686e16..e00b5ac 100644 --- a/helix-core/src/test/java/org/apache/helix/messaging/TestDefaultMessagingService.java +++ b/helix-core/src/test/java/org/apache/helix/messaging/TestDefaultMessagingService.java @@ -33,13 +33,13 @@ import org.apache.helix.NotificationContext; import org.apache.helix.PropertyKey; import org.apache.helix.PropertyType; import org.apache.helix.ZNRecord; +import org.apache.helix.controller.strategy.DefaultTwoStateStrategy; import org.apache.helix.messaging.handling.HelixTaskResult; import org.apache.helix.messaging.handling.MessageHandler; import org.apache.helix.messaging.handling.MessageHandlerFactory; import org.apache.helix.model.ExternalView; import org.apache.helix.model.LiveInstance.LiveInstanceProperty; import org.apache.helix.model.Message; -import org.apache.helix.tools.DefaultIdealStateCalculator; import org.testng.AssertJUnit; import org.testng.annotations.Test; @@ -94,7 +94,7 @@ public class TestDefaultMessagingService { _liveInstances.add(metaData); } _externalView = - DefaultIdealStateCalculator.calculateIdealState(_instances, _partitions, _replicas, _db, + DefaultTwoStateStrategy.calculateIdealState(_instances, _partitions, _replicas, _db, "MASTER", "SLAVE"); } http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterStatusMonitor.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterStatusMonitor.java b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterStatusMonitor.java index 711aff2..facb4ea 100644 --- a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterStatusMonitor.java +++ b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterStatusMonitor.java @@ -29,8 +29,8 @@ import org.apache.helix.Mocks; import org.apache.helix.NotificationContext; import org.apache.helix.PropertyType; import org.apache.helix.ZNRecord; +import org.apache.helix.controller.strategy.DefaultTwoStateStrategy; import org.apache.helix.model.LiveInstance.LiveInstanceProperty; -import org.apache.helix.tools.DefaultIdealStateCalculator; import org.testng.annotations.Test; public class TestClusterStatusMonitor { @@ -50,13 +50,11 @@ public class TestClusterStatusMonitor { _instances.add(instance); } ZNRecord externalView = - DefaultIdealStateCalculator.calculateIdealState(_instances, _partitions, _replicas, _db, + DefaultTwoStateStrategy.calculateIdealState(_instances, _partitions, _replicas, _db, "MASTER", "SLAVE"); ZNRecord externalView2 = - DefaultIdealStateCalculator.calculateIdealState(_instances, 80, 2, _db2, "MASTER", - "SLAVE"); - + DefaultTwoStateStrategy.calculateIdealState(_instances, 80, 2, _db2, "MASTER", "SLAVE"); } public ZNRecord getProperty(PropertyType type, String resource) { @@ -100,12 +98,11 @@ public class TestClusterStatusMonitor { _liveInstances.add(metaData); } ZNRecord externalView = - DefaultIdealStateCalculator.calculateIdealState(_instances, _partitions, _replicas, _db, + DefaultTwoStateStrategy.calculateIdealState(_instances, _partitions, _replicas, _db, "MASTER", "SLAVE"); ZNRecord externalView2 = - DefaultIdealStateCalculator.calculateIdealState(_instances, 80, 2, "TestDB", "MASTER", - "SLAVE"); + DefaultTwoStateStrategy.calculateIdealState(_instances, 80, 2, "TestDB", "MASTER", "SLAVE"); List externalViews = new ArrayList(); externalViews.add(externalView); http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/21c4fcb5/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestResourceMonitor.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestResourceMonitor.java b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestResourceMonitor.java index d631dd2..6712c40 100644 --- a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestResourceMonitor.java +++ b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestResourceMonitor.java @@ -31,11 +31,11 @@ import org.apache.helix.PropertyKey; import org.apache.helix.PropertyType; import org.apache.helix.ZNRecord; import org.apache.helix.PropertyKey.Builder; +import org.apache.helix.controller.strategy.DefaultTwoStateStrategy; import org.apache.helix.model.ExternalView; import org.apache.helix.model.IdealState; import org.apache.helix.model.LiveInstance.LiveInstanceProperty; import org.apache.helix.monitoring.mbeans.ResourceMonitor; -import org.apache.helix.tools.DefaultIdealStateCalculator; import org.testng.AssertJUnit; import org.testng.annotations.Test; @@ -94,8 +94,8 @@ public class TestResourceMonitor { } _idealState = - DefaultIdealStateCalculator.calculateIdealState(_instances, _partitions, _replicas, - _dbName, "MASTER", "SLAVE"); + DefaultTwoStateStrategy.calculateIdealState(_instances, _partitions, _replicas, _dbName, + "MASTER", "SLAVE"); _externalView = new ZNRecord(_idealState); }