tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jihoon...@apache.org
Subject tajo git commit: TAJO-1801: Table name is not unique of tableDescMap in QueryMasterTask.
Date Thu, 27 Aug 2015 16:37:30 GMT
Repository: tajo
Updated Branches:
  refs/heads/master 4be674610 -> 848a8c3d5


TAJO-1801: Table name is not unique of tableDescMap in QueryMasterTask.

Closes #710


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/848a8c3d
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/848a8c3d
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/848a8c3d

Branch: refs/heads/master
Commit: 848a8c3d58fa219dee09baa808191d65fff61ffe
Parents: 4be6746
Author: Jihoon Son <jihoonson@apache.org>
Authored: Fri Aug 28 01:36:55 2015 +0900
Committer: Jihoon Son <jihoonson@apache.org>
Committed: Fri Aug 28 01:36:55 2015 +0900

----------------------------------------------------------------------
 CHANGES                                               |  5 ++++-
 .../main/java/org/apache/tajo/querymaster/Query.java  | 12 ++++--------
 .../org/apache/tajo/querymaster/QueryMasterTask.java  | 12 ++++++------
 .../org/apache/tajo/querymaster/Repartitioner.java    |  6 +++---
 .../main/java/org/apache/tajo/querymaster/Stage.java  | 14 ++++++--------
 5 files changed, 23 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/848a8c3d/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 32e84b5..8fcc1d0 100644
--- a/CHANGES
+++ b/CHANGES
@@ -234,6 +234,9 @@ Release 0.11.0 - unreleased
 
   BUG FIXES
 
+    TAJO-1801: Table name is not unique of tableDescMap in QueryMasterTask. 
+    (jihoon)
+
     TAJO-1798: Dynamic partitioning occasionally fails. (jaehwa)
     
     TAJO-1799: Fix incorrect event handler when kill-query failed. (jinho)
@@ -250,7 +253,7 @@ Release 0.11.0 - unreleased
 
     TAJO-1776: Fix Invalid column type in JDBC. (jinho)
 
-    TAJO-1781: Join condition is still not found when it exists in OR clause. i
+    TAJO-1781: Join condition is still not found when it exists in OR clause. 
     (jihoon)
 
     TAJO-1777: JsonLineDeserializer returns invalid unicode text, 

http://git-wip-us.apache.org/repos/asf/tajo/blob/848a8c3d/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
index 9560353..8bb3845 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
@@ -35,26 +35,22 @@ import org.apache.tajo.QueryVars;
 import org.apache.tajo.SessionVars;
 import org.apache.tajo.TajoProtos.QueryState;
 import org.apache.tajo.catalog.*;
-import org.apache.tajo.catalog.proto.CatalogProtos.UpdateTableStatsProto;
 import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto;
-import org.apache.tajo.catalog.CatalogService;
-import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.proto.CatalogProtos.UpdateTableStatsProto;
 import org.apache.tajo.catalog.statistics.TableStats;
 import org.apache.tajo.conf.TajoConf;
 import org.apache.tajo.engine.planner.global.ExecutionBlock;
 import org.apache.tajo.engine.planner.global.ExecutionBlockCursor;
 import org.apache.tajo.engine.planner.global.ExecutionQueue;
 import org.apache.tajo.engine.planner.global.MasterPlan;
-import org.apache.tajo.exception.TajoException;
-import org.apache.tajo.exception.TajoInternalError;
-import org.apache.tajo.plan.logical.*;
 import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.exception.TajoInternalError;
 import org.apache.tajo.master.event.*;
+import org.apache.tajo.plan.logical.*;
 import org.apache.tajo.plan.util.PlannerUtil;
 import org.apache.tajo.storage.StorageConstants;
-import org.apache.tajo.storage.TablespaceManager;
 import org.apache.tajo.storage.Tablespace;
+import org.apache.tajo.storage.TablespaceManager;
 import org.apache.tajo.util.TUtil;
 import org.apache.tajo.util.history.QueryHistory;
 import org.apache.tajo.util.history.StageHistory;

http://git-wip-us.apache.org/repos/asf/tajo/blob/848a8c3d/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
index 52e0a96..1313dad 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
@@ -91,7 +91,7 @@ public class QueryMasterTask extends CompositeService {
 
   private final long querySubmitTime;
 
-  private Map<String, TableDesc> tableDescMap = new HashMap<String, TableDesc>();
+  private final Map<Integer, TableDesc> tableDescMap = new HashMap<>();
 
   private TajoConf systemConf;
 
@@ -332,7 +332,7 @@ public class QueryMasterTask extends CompositeService {
         if (scanNodes != null) {
           for (LogicalNode eachScanNode : scanNodes) {
             ScanNode scanNode = (ScanNode) eachScanNode;
-            tableDescMap.put(scanNode.getCanonicalName(), scanNode.getTableDesc());
+            tableDescMap.put(scanNode.getPID(), scanNode.getTableDesc());
           }
         }
 
@@ -340,7 +340,7 @@ public class QueryMasterTask extends CompositeService {
         if (scanNodes != null) {
           for (LogicalNode eachScanNode : scanNodes) {
             ScanNode scanNode = (ScanNode) eachScanNode;
-            tableDescMap.put(scanNode.getCanonicalName(), scanNode.getTableDesc());
+            tableDescMap.put(scanNode.getPID(), scanNode.getTableDesc());
           }
         }
 
@@ -348,7 +348,7 @@ public class QueryMasterTask extends CompositeService {
         if (scanNodes != null) {
           for (LogicalNode eachScanNode : scanNodes) {
             ScanNode scanNode = (ScanNode) eachScanNode;
-            tableDescMap.put(scanNode.getCanonicalName(), scanNode.getTableDesc());
+            tableDescMap.put(scanNode.getPID(), scanNode.getTableDesc());
           }
         }
       }
@@ -533,8 +533,8 @@ public class QueryMasterTask extends CompositeService {
       return query.getStage(id);
     }
 
-    public Map<String, TableDesc> getTableDescMap() {
-      return tableDescMap;
+    public TableDesc getTableDesc(ScanNode scanNode) {
+      return tableDescMap.get(scanNode.getPID());
     }
 
     public float getProgress() {

http://git-wip-us.apache.org/repos/asf/tajo/blob/848a8c3d/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
index 0d5880e..c4fc645 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
@@ -88,7 +88,7 @@ public class Repartitioner {
 
     // initialize variables from the child operators
     for (int i = 0; i < scans.length; i++) {
-      TableDesc tableDesc = masterContext.getTableDescMap().get(scans[i].getCanonicalName());
+      TableDesc tableDesc = masterContext.getTableDesc(scans[i]);
 
       if (tableDesc == null) { // if it is a real table stored on storage
         if (execBlock.getUnionScanMap() != null && !execBlock.getUnionScanMap().isEmpty())
{
@@ -376,7 +376,7 @@ public class Repartitioner {
       for (ScanNode eachScan: broadcastScans) {
 
         Path[] partitionScanPaths = null;
-        TableDesc tableDesc = masterContext.getTableDescMap().get(eachScan.getCanonicalName());
+        TableDesc tableDesc = masterContext.getTableDesc(eachScan);
         Tablespace space = TablespaceManager.get(tableDesc.getUri()).get();
 
         if (eachScan.getType() == NodeType.PARTITIONS_SCAN) {
@@ -498,7 +498,7 @@ public class Repartitioner {
     List<Fragment> broadcastFragments = new ArrayList<Fragment>();
     for (int i = 0; i < scans.length; i++) {
       ScanNode scan = scans[i];
-      TableDesc desc = stage.getContext().getTableDescMap().get(scan.getCanonicalName());
+      TableDesc desc = stage.getContext().getTableDesc(scan);
       TableMeta meta = desc.getMeta();
 
       Collection<Fragment> scanFragments;

http://git-wip-us.apache.org/repos/asf/tajo/blob/848a8c3d/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
index f6c9cdb..12e4366 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
@@ -31,7 +31,6 @@ import org.apache.tajo.catalog.CatalogUtil;
 import org.apache.tajo.catalog.Schema;
 import org.apache.tajo.catalog.TableDesc;
 import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos;
 import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto;
 import org.apache.tajo.catalog.statistics.ColumnStats;
 import org.apache.tajo.catalog.statistics.StatisticsUtil;
@@ -44,12 +43,12 @@ import org.apache.tajo.engine.planner.global.ExecutionBlock;
 import org.apache.tajo.engine.planner.global.MasterPlan;
 import org.apache.tajo.exception.TajoException;
 import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.MultipleAggregationStage;
-import org.apache.tajo.plan.serder.PlanProto.EnforceProperty;
 import org.apache.tajo.master.TaskState;
 import org.apache.tajo.master.event.*;
 import org.apache.tajo.master.event.TaskAttemptToSchedulerEvent.TaskAttemptScheduleContext;
 import org.apache.tajo.plan.logical.*;
+import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.MultipleAggregationStage;
+import org.apache.tajo.plan.serder.PlanProto.EnforceProperty;
 import org.apache.tajo.plan.util.PlannerUtil;
 import org.apache.tajo.querymaster.Task.IntermediateEntry;
 import org.apache.tajo.rpc.AsyncRpcClient;
@@ -57,8 +56,8 @@ import org.apache.tajo.rpc.NullCallback;
 import org.apache.tajo.rpc.RpcClientManager;
 import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
 import org.apache.tajo.storage.FileTablespace;
-import org.apache.tajo.storage.TablespaceManager;
 import org.apache.tajo.storage.Tablespace;
+import org.apache.tajo.storage.TablespaceManager;
 import org.apache.tajo.storage.fragment.Fragment;
 import org.apache.tajo.unit.StorageUnit;
 import org.apache.tajo.util.KeyValueSet;
@@ -76,9 +75,9 @@ import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import static org.apache.tajo.ResourceProtos.*;
 import static org.apache.tajo.conf.TajoConf.ConfVars;
 import static org.apache.tajo.plan.serder.PlanProto.ShuffleType;
-import static org.apache.tajo.ResourceProtos.*;
 
 
 /**
@@ -1058,12 +1057,11 @@ public class Stage implements EventHandler<StageEvent> {
 
     public static long getInputVolume(MasterPlan masterPlan, QueryMasterTask.QueryMasterTaskContext
context,
                                       ExecutionBlock execBlock) {
-      Map<String, TableDesc> tableMap = context.getTableDescMap();
       if (masterPlan.isLeaf(execBlock)) {
         ScanNode[] outerScans = execBlock.getScanNodes();
         long maxVolume = 0;
         for (ScanNode eachScanNode: outerScans) {
-          TableStats stat = tableMap.get(eachScanNode.getCanonicalName()).getStats();
+          TableStats stat = context.getTableDesc(eachScanNode).getStats();
           if (stat.getNumBytes() > maxVolume) {
             maxVolume = stat.getNumBytes();
           }
@@ -1089,7 +1087,7 @@ public class Stage implements EventHandler<StageEvent> {
       ScanNode[] scans = execBlock.getScanNodes();
       Preconditions.checkArgument(scans.length == 1, "Must be Scan Query");
       ScanNode scan = scans[0];
-      TableDesc table = stage.context.getTableDescMap().get(scan.getCanonicalName());
+      TableDesc table = stage.context.getTableDesc(scan);
 
       Collection<Fragment> fragments;
       Tablespace tablespace = TablespaceManager.get(scan.getTableDesc().getUri()).get();


Mime
View raw message