carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [04/24] carbondata git commit: performance issue resolved
Date Wed, 12 Jul 2017 15:09:16 GMT
performance issue resolved


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1278c41b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1278c41b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1278c41b

Branch: refs/heads/streaming_ingest
Commit: 1278c41bb1692a3e542194626346c3324a5cfb4e
Parents: 427b88b
Author: sahakushal <carbondatadeveloper@gmail.com>
Authored: Wed Jun 28 16:29:01 2017 +0530
Committer: Venkata Ramana G <ramana.gollamudi@huawei.com>
Committed: Wed Jul 5 10:38:56 2017 +0530

----------------------------------------------------------------------
 .../apache/carbondata/spark/load/CarbonLoaderUtil.java    | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/1278c41b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index 54e12f3..5b603aa 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -742,14 +742,20 @@ public final class CarbonLoaderUtil {
       if (null == nodeAndBlockMapping.get(node)) {
         list = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
         list.add(nbr.getBlock());
-        Collections.sort(list);
         nodeAndBlockMapping.put(node, list);
       } else {
         list = nodeAndBlockMapping.get(node);
         list.add(nbr.getBlock());
-        Collections.sort(list);
       }
     }
+    /*for resolving performance issue, removed values() with entrySet () iterating the values
and
+    sorting it.entrySet will give the logical view for hashMap and we dont query the map
twice for
+    each key whereas values () iterate twice*/
+    Iterator<Map.Entry<String, List<Distributable>>> iterator =
+        nodeAndBlockMapping.entrySet().iterator();
+    while (iterator.hasNext()) {
+      Collections.sort(iterator.next().getValue());
+    }
   }
 
   /**


Mime
View raw message