carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vimal...@apache.org
Subject [1/2] incubator-carbondata git commit: combine two same name classes
Date Wed, 07 Sep 2016 01:50:53 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 005186223 -> 4aa22415d


combine two same name classes


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/5675728f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/5675728f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/5675728f

Branch: refs/heads/master
Commit: 5675728ff193167ff298f510f14b8b0edf780417
Parents: 0051862
Author: Zuo Wang <wangzuo.nj@qq.com>
Authored: Tue Sep 6 14:48:33 2016 +0800
Committer: Vimal-Das <vimaldas@apache.org>
Committed: Wed Sep 7 07:13:18 2016 +0530

----------------------------------------------------------------------
 .../spark/load/CarbonLoaderUtilTest.java        | 162 +++++++++++++++-
 .../spark/load/CarbonLoaderUtilTest.java        | 191 -------------------
 2 files changed, 160 insertions(+), 193 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5675728f/integration/spark/src/test/java/org/carbondata/integration/spark/load/CarbonLoaderUtilTest.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/java/org/carbondata/integration/spark/load/CarbonLoaderUtilTest.java
b/integration/spark/src/test/java/org/carbondata/integration/spark/load/CarbonLoaderUtilTest.java
index 81c8f0c..0161a45 100644
--- a/integration/spark/src/test/java/org/carbondata/integration/spark/load/CarbonLoaderUtilTest.java
+++ b/integration/spark/src/test/java/org/carbondata/integration/spark/load/CarbonLoaderUtilTest.java
@@ -20,19 +20,177 @@ package org.apache.carbondata.integration.spark.load;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
+import org.apache.carbondata.core.carbon.datastore.block.Distributable;
 import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
 import org.apache.carbondata.spark.load.CarbonLoaderUtil;
+import org.junit.Assert;
 import org.junit.Test;
-import org.pentaho.di.core.util.Assert;
 
 /**
- * Test class for LoaderUtil Test
+ * Test class to test block distribution functionality
  */
 public class CarbonLoaderUtilTest {
+  List<Distributable> blockInfos = null;
+  int noOfNodesInput = -1;
+  List<String> activeNode = null;
+  Map<String, List<Distributable>> expected = null;
+  Map<String, List<Distributable>> mapOfNodes = null;
+
+  @Test public void nodeBlockMapping() throws Exception {
+
+    // scenario when the 3 nodes and 3 executors
+    initSet1();
+    Map<String, List<Distributable>> mapOfNodes =
+            CarbonLoaderUtil.nodeBlockMapping(blockInfos, noOfNodesInput, activeNode);
+    // node allocation
+    Assert.assertTrue("Node Allocation", expected.size() == mapOfNodes.size());
+    // block allocation
+    boolean isEqual = compareResult(expected, mapOfNodes);
+    Assert.assertTrue("Block Allocation", isEqual);
+
+    // 2 node and 3 executors
+    initSet2();
+    mapOfNodes = CarbonLoaderUtil.nodeBlockMapping(blockInfos, noOfNodesInput, activeNode);
+    // node allocation
+    Assert.assertTrue("Node Allocation", expected.size() == mapOfNodes.size());
+    // block allocation
+    isEqual = compareResult(expected, mapOfNodes);
+    Assert.assertTrue("Block Allocation", isEqual);
+
+    // 3 data node and 2 executors
+    initSet3();
+    mapOfNodes = CarbonLoaderUtil.nodeBlockMapping(blockInfos, noOfNodesInput, activeNode);
+    // node allocation
+    Assert.assertTrue("Node Allocation", expected.size() == mapOfNodes.size());
+    // block allocation
+    isEqual = compareResult(expected, mapOfNodes);
+    Assert.assertTrue("Block Allocation", isEqual);
+  }
+
+  /**
+   * compares the blocks allocation
+   *
+   * @param expectedResult
+   * @param actualResult
+   * @return
+   */
+  private boolean compareResult(Map<String, List<Distributable>> expectedResult,
+                                Map<String, List<Distributable>> actualResult)
{
+    expectedResult = sortByListSize(expectedResult);
+    actualResult = sortByListSize(actualResult);
+    List<List<Distributable>> expectedList = new LinkedList(expectedResult.entrySet());
+    List<List<Distributable>> mapOfNodesList = new LinkedList(actualResult.entrySet());
+    boolean isEqual = expectedList.size() == mapOfNodesList.size();
+    if (isEqual) {
+      for (int i = 0; i < expectedList.size(); i++) {
+        int size1 = ((List) ((Map.Entry) (expectedList.get(i))).getValue()).size();
+        int size2 = ((List) ((Map.Entry) (mapOfNodesList.get(i))).getValue()).size();
+        isEqual = size1 == size2;
+        if (!isEqual) {
+          break;
+        }
+      }
+    }
+    return isEqual;
+  }
+
+  /**
+   * sort by list size
+   *
+   * @param map
+   * @return
+   */
+  private static Map<String, List<Distributable>> sortByListSize(
+          Map<String, List<Distributable>> map) {
+    List<List<Distributable>> list = new LinkedList(map.entrySet());
+    Collections.sort(list, new Comparator() {
+      public int compare(Object obj1, Object obj2) {
+        if (obj1 == null && obj2 == null) {
+          return 0;
+        } else if (obj1 == null) {
+          return 1;
+        } else if (obj2 == null) {
+          return -1;
+        }
+        int size1 = ((List) ((Map.Entry) (obj1)).getValue()).size();
+        int size2 = ((List) ((Map.Entry) (obj2)).getValue()).size();
+        return size2 - size1;
+      }
+    });
+
+    Map res = new LinkedHashMap();
+    for (Iterator it = list.iterator(); it.hasNext(); ) {
+      Map.Entry entry = (Map.Entry) it.next();
+      res.put(entry.getKey(), entry.getValue());
+    }
+    return res;
+  }
+
+  void initSet1() {
+    blockInfos = new ArrayList<>();
+    activeNode = new ArrayList<>();
+    activeNode.add("node-7");
+    activeNode.add("node-9");
+    activeNode.add("node-11");
+    String[] location = { "node-7", "node-9", "node-11" };
+    blockInfos.add(new TableBlockInfo("node", 1, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 2, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 3, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 4, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 5, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 6, "1", location, 0));
+    expected = new HashMap<>();
+    expected.put("node-7", blockInfos.subList(0, 2));
+    expected.put("node-9", blockInfos.subList(2, 4));
+    expected.put("node-11", blockInfos.subList(4, 6));
+  }
+
+  void initSet2() {
+    blockInfos = new ArrayList<>();
+    activeNode = new ArrayList<>();
+    activeNode.add("node-7");
+    activeNode.add("node-9");
+    activeNode.add("node-11");
+    String[] location = { "node-7", "node-11" };
+    blockInfos.add(new TableBlockInfo("node", 1, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 2, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 3, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 4, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 5, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 6, "1", location, 0));
+    expected = new HashMap<>();
+    expected.put("node-7", blockInfos.subList(0, 2));
+    expected.put("node-9", blockInfos.subList(2, 4));
+    expected.put("node-11", blockInfos.subList(4, 6));
+  }
+
+  void initSet3() {
+    blockInfos = new ArrayList<>();
+    activeNode = new ArrayList<>();
+    activeNode.add("node-7");
+    activeNode.add("node-11");
+    String[] location = { "node-7", "node-9", "node-11" };
+    blockInfos.add(new TableBlockInfo("node", 1, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 2, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 3, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 4, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 5, "1", location, 0));
+    blockInfos.add(new TableBlockInfo("node", 6, "1", location, 0));
+    expected = new HashMap<>();
+    expected.put("node-7", blockInfos.subList(0, 3));
+    expected.put("node-11", blockInfos.subList(3, 6));
+  }
+
 
   /**
    * Test case with 4 blocks and 4 nodes with 3 replication.

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5675728f/integration/spark/src/test/scala/org/apache/carbondata/spark/load/CarbonLoaderUtilTest.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/load/CarbonLoaderUtilTest.java
b/integration/spark/src/test/scala/org/apache/carbondata/spark/load/CarbonLoaderUtilTest.java
deleted file mode 100644
index f8ec400..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/load/CarbonLoaderUtilTest.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.spark.load;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.carbondata.core.carbon.datastore.block.Distributable;
-import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test class to test block distribution functionality
- */
-public class CarbonLoaderUtilTest {
-  List<Distributable> blockInfos = null;
-  int noOfNodesInput = -1;
-  List<String> activeNode = null;
-  Map<String, List<Distributable>> expected = null;
-  Map<String, List<Distributable>> mapOfNodes = null;
-
-  @Test public void nodeBlockMapping() throws Exception {
-
-    // scenario when the 3 nodes and 3 executors
-    initSet1();
-    Map<String, List<Distributable>> mapOfNodes =
-        CarbonLoaderUtil.nodeBlockMapping(blockInfos, noOfNodesInput, activeNode);
-    // node allocation
-    Assert.assertTrue("Node Allocation", expected.size() == mapOfNodes.size());
-    // block allocation
-    boolean isEqual = compareResult(expected, mapOfNodes);
-    Assert.assertTrue("Block Allocation", isEqual);
-
-    // 2 node and 3 executors
-    initSet2();
-    mapOfNodes = CarbonLoaderUtil.nodeBlockMapping(blockInfos, noOfNodesInput, activeNode);
-    // node allocation
-    Assert.assertTrue("Node Allocation", expected.size() == mapOfNodes.size());
-    // block allocation
-    isEqual = compareResult(expected, mapOfNodes);
-    Assert.assertTrue("Block Allocation", isEqual);
-
-    // 3 data node and 2 executors
-    initSet3();
-    mapOfNodes = CarbonLoaderUtil.nodeBlockMapping(blockInfos, noOfNodesInput, activeNode);
-    // node allocation
-    Assert.assertTrue("Node Allocation", expected.size() == mapOfNodes.size());
-    // block allocation
-    isEqual = compareResult(expected, mapOfNodes);
-    Assert.assertTrue("Block Allocation", isEqual);
-  }
-
-  /**
-   * compares the blocks allocation
-   *
-   * @param expectedResult
-   * @param actualResult
-   * @return
-   */
-  private boolean compareResult(Map<String, List<Distributable>> expectedResult,
-      Map<String, List<Distributable>> actualResult) {
-    expectedResult = sortByListSize(expectedResult);
-    actualResult = sortByListSize(actualResult);
-    List<List<Distributable>> expectedList = new LinkedList(expectedResult.entrySet());
-    List<List<Distributable>> mapOfNodesList = new LinkedList(actualResult.entrySet());
-    boolean isEqual = expectedList.size() == mapOfNodesList.size();
-    if (isEqual) {
-      for (int i = 0; i < expectedList.size(); i++) {
-        int size1 = ((List) ((Map.Entry) (expectedList.get(i))).getValue()).size();
-        int size2 = ((List) ((Map.Entry) (mapOfNodesList.get(i))).getValue()).size();
-        isEqual = size1 == size2;
-        if (!isEqual) {
-          break;
-        }
-      }
-    }
-    return isEqual;
-  }
-
-  /**
-   * sort by list size
-   *
-   * @param map
-   * @return
-   */
-  private static Map<String, List<Distributable>> sortByListSize(
-      Map<String, List<Distributable>> map) {
-    List<List<Distributable>> list = new LinkedList(map.entrySet());
-    Collections.sort(list, new Comparator() {
-      public int compare(Object obj1, Object obj2) {
-        if (obj1 == null && obj2 == null) {
-          return 0;
-        } else if (obj1 == null) {
-          return 1;
-        } else if (obj2 == null) {
-          return -1;
-        }
-        int size1 = ((List) ((Map.Entry) (obj1)).getValue()).size();
-        int size2 = ((List) ((Map.Entry) (obj2)).getValue()).size();
-        return size2 - size1;
-      }
-    });
-
-    Map res = new LinkedHashMap();
-    for (Iterator it = list.iterator(); it.hasNext(); ) {
-      Map.Entry entry = (Map.Entry) it.next();
-      res.put(entry.getKey(), entry.getValue());
-    }
-    return res;
-  }
-
-  void initSet1() {
-    blockInfos = new ArrayList<>();
-    activeNode = new ArrayList<>();
-    activeNode.add("node-7");
-    activeNode.add("node-9");
-    activeNode.add("node-11");
-    String[] location = { "node-7", "node-9", "node-11" };
-    blockInfos.add(new TableBlockInfo("node", 1, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 2, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 3, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 4, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 5, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 6, "1", location, 0));
-    expected = new HashMap<>();
-    expected.put("node-7", blockInfos.subList(0, 2));
-    expected.put("node-9", blockInfos.subList(2, 4));
-    expected.put("node-11", blockInfos.subList(4, 6));
-  }
-
-  void initSet2() {
-    blockInfos = new ArrayList<>();
-    activeNode = new ArrayList<>();
-    activeNode.add("node-7");
-    activeNode.add("node-9");
-    activeNode.add("node-11");
-    String[] location = { "node-7", "node-11" };
-    blockInfos.add(new TableBlockInfo("node", 1, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 2, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 3, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 4, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 5, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 6, "1", location, 0));
-    expected = new HashMap<>();
-    expected.put("node-7", blockInfos.subList(0, 2));
-    expected.put("node-9", blockInfos.subList(2, 4));
-    expected.put("node-11", blockInfos.subList(4, 6));
-  }
-
-  void initSet3() {
-    blockInfos = new ArrayList<>();
-    activeNode = new ArrayList<>();
-    activeNode.add("node-7");
-    activeNode.add("node-11");
-    String[] location = { "node-7", "node-9", "node-11" };
-    blockInfos.add(new TableBlockInfo("node", 1, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 2, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 3, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 4, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 5, "1", location, 0));
-    blockInfos.add(new TableBlockInfo("node", 6, "1", location, 0));
-    expected = new HashMap<>();
-    expected.put("node-7", blockInfos.subList(0, 3));
-    expected.put("node-11", blockInfos.subList(3, 6));
-  }
-}
\ No newline at end of file


Mime
View raw message