tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jihoon...@apache.org
Subject tajo git commit: TAJO-1713: Change the type of edge cache in JoinGraphContext from HashMap to LRUMap.
Date Thu, 30 Jul 2015 01:25:22 GMT
Repository: tajo
Updated Branches:
  refs/heads/master 18b898ffb -> 97e61e6f4


TAJO-1713: Change the type of edge cache in JoinGraphContext from HashMap to LRUMap.

Closes #654


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/97e61e6f
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/97e61e6f
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/97e61e6f

Branch: refs/heads/master
Commit: 97e61e6f4e9ff1ebeac388dd22e139e1fd486e99
Parents: 18b898f
Author: Jihoon Son <jihoonson@apache.org>
Authored: Thu Jul 30 10:24:52 2015 +0900
Committer: Jihoon Son <jihoonson@apache.org>
Committed: Thu Jul 30 10:24:52 2015 +0900

----------------------------------------------------------------------
 CHANGES                                         |  3 +++
 .../tajo/util/graph/SimpleDirectedGraph.java    | 11 +++++++++
 .../planner/physical/TestBSTIndexExec.java      |  1 +
 .../engine/planner/physical/TestSortExec.java   |  1 +
 .../org/apache/tajo/plan/LogicalOptimizer.java  |  4 +--
 .../apache/tajo/plan/joinorder/JoinGraph.java   |  1 -
 .../tajo/plan/joinorder/JoinGraphContext.java   | 26 +++++++++++++++++---
 7 files changed, 40 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/97e61e6f/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 4632ddf..2b9ea92 100644
--- a/CHANGES
+++ b/CHANGES
@@ -373,6 +373,9 @@ Release 0.11.0 - unreleased
   
   TASKS
 
+    TAJO-1713: Change the type of edge cache in JoinGraphContext from 
+    HashMap to LRUMap. (jihoon)
+
     TAJO-1273: Merge DirectRawFile to master branch. (jinho)
 
     TAJO-1628: Add a documentation for join operation. (jihoon)

http://git-wip-us.apache.org/repos/asf/tajo/blob/97e61e6f/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java
b/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java
index d72a2dd..045add5 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java
@@ -37,6 +37,17 @@ public class SimpleDirectedGraph<V, E> implements DirectedGraph<V,E>
{
   /** map: parent -> child */
   protected Map<V, Map<V, E>> reversedEdges = TUtil.newLinkedHashMap();
 
+  public void clear() {
+    for (Map<V, E> eachEdge : directedEdges.values()) {
+      eachEdge.clear();
+    }
+    for (Map<V, E> eachEdge : reversedEdges.values()) {
+      eachEdge.clear();
+    }
+    directedEdges.clear();
+    reversedEdges.clear();
+  }
+
   @Override
   public int getVertexSize() {
     return directedEdges.size();

http://git-wip-us.apache.org/repos/asf/tajo/blob/97e61e6f/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
index 036ca78..6fb7a45 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
@@ -83,6 +83,7 @@ public class TestBSTIndexExec {
   public void setup() throws Exception {
     this.randomValues = new HashMap<Integer, Integer>();
     this.conf = new TajoConf();
+    conf.set(CommonTestingUtil.TAJO_TEST_KEY, CommonTestingUtil.TAJO_TEST_TRUE);
     util = new TajoTestingCluster();
     util.startCatalogCluster();
     catalog = util.getMiniCatalogCluster().getCatalog();

http://git-wip-us.apache.org/repos/asf/tajo/blob/97e61e6f/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
index 743c51d..1c55d10 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
@@ -72,6 +72,7 @@ public class TestSortExec {
   @BeforeClass
   public static void setUp() throws Exception {
     conf = new TajoConf();
+    conf.set(CommonTestingUtil.TAJO_TEST_KEY, CommonTestingUtil.TAJO_TEST_TRUE);
     util = TpchTestBase.getInstance().getTestingCluster();
     catalog = util.getMaster().getCatalog();
     workDir = CommonTestingUtil.getTestDir(TEST_PATH);

http://git-wip-us.apache.org/repos/asf/tajo/blob/97e61e6f/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
index 2bfa4a7..fe49994 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
@@ -32,7 +32,6 @@ import org.apache.tajo.exception.TajoException;
 import org.apache.tajo.plan.expr.AlgebraicUtil;
 import org.apache.tajo.plan.expr.EvalNode;
 import org.apache.tajo.plan.expr.EvalTreeUtil;
-import org.apache.tajo.plan.expr.EvalType;
 import org.apache.tajo.plan.joinorder.*;
 import org.apache.tajo.plan.logical.*;
 import org.apache.tajo.plan.rewrite.BaseLogicalPlanRewriteEngine;
@@ -60,7 +59,6 @@ public class LogicalOptimizer {
   private JoinOrderAlgorithm joinOrderAlgorithm = new GreedyHeuristicJoinOrderAlgorithm();
 
   public LogicalOptimizer(TajoConf conf) {
-
     Class clazz = conf.getClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS);
     LogicalPlanRewriteRuleProvider provider = (LogicalPlanRewriteRuleProvider) ReflectionUtil.newInstance(clazz,
conf);
 
@@ -129,6 +127,8 @@ public class LogicalOptimizer {
       String optimizedOrder = JoinOrderStringBuilder.buildJoinOrderString(plan, block);
       block.addPlanHistory("Non-optimized join order: " + originalOrder + " (cost: " + nonOptimizedJoinCost
+ ")");
       block.addPlanHistory("Optimized join order    : " + optimizedOrder + " (cost: " + order.getCost()
+ ")");
+
+      joinGraphContext.clear();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/97e61e6f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
index 7687474..8fcdce7 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
@@ -18,7 +18,6 @@
 
 package org.apache.tajo.plan.joinorder;
 
-import org.apache.tajo.plan.PlanningException;
 import org.apache.tajo.plan.logical.JoinSpec;
 import org.apache.tajo.plan.util.PlannerUtil;
 import org.apache.tajo.util.graph.SimpleUndirectedGraph;

http://git-wip-us.apache.org/repos/asf/tajo/blob/97e61e6f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java
b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java
index 97cd569..a3b529c 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java
@@ -18,13 +18,13 @@
 
 package org.apache.tajo.plan.joinorder;
 
+import org.apache.commons.collections.map.LRUMap;
 import org.apache.tajo.plan.expr.EvalNode;
 import org.apache.tajo.plan.logical.JoinSpec;
 import org.apache.tajo.util.Pair;
 import org.apache.tajo.util.TUtil;
 
 import java.util.Collection;
-import java.util.Map;
 import java.util.Set;
 
 public class JoinGraphContext {
@@ -32,8 +32,8 @@ public class JoinGraphContext {
   private JoinGraph joinGraph = new JoinGraph();
 
   // New join edges are frequently created during join order optimization.
-  // This cache is to reduce the overhead of join edge creation.
-  private Map<Pair<JoinVertex,JoinVertex>, JoinEdge> edgeCache = TUtil.newHashMap();
+  // This cache is to reduce such overhead.
+  private LRUMap edgeCache = new LRUMap(10000);
 
   // candidate predicates contain the predicates which are not pushed to any join nodes yet.
   // evaluated predicates contain the predicates which are already pushed to some join nodes.
@@ -129,9 +129,27 @@ public class JoinGraphContext {
   public JoinEdge getCachedOrNewJoinEdge(JoinSpec joinSpec, JoinVertex left, JoinVertex right)
{
     Pair<JoinVertex,JoinVertex> cacheKey = new Pair<JoinVertex, JoinVertex>(left,
right);
     if (edgeCache.containsKey(cacheKey)) {
-      return edgeCache.get(cacheKey);
+      return (JoinEdge) edgeCache.get(cacheKey);
     } else {
       return cacheEdge(new JoinEdge(joinSpec, left, right));
     }
   }
+
+  public void clear() {
+    rootVertexes.clear();
+    candidateJoinConditions.clear();
+    candidateJoinFilters.clear();
+    evaluatedJoinConditions.clear();
+    evaluatedJoinFilters.clear();
+    edgeCache.clear();
+    joinGraph.clear();
+
+    rootVertexes = null;
+    candidateJoinConditions = null;
+    candidateJoinFilters = null;
+    evaluatedJoinConditions = null;
+    evaluatedJoinFilters = null;
+    edgeCache = null;
+    joinGraph = null;
+  }
 }
\ No newline at end of file


Mime
View raw message