tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hj...@apache.org
Subject [26/29] tajo git commit: TAJO-1231: Implicit table properties in session is not stored in table property.
Date Fri, 05 Dec 2014 08:21:30 GMT
TAJO-1231: Implicit table properties in session is not stored in table property.

Closes #287


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/ab2efce8
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/ab2efce8
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/ab2efce8

Branch: refs/heads/hbase_storage
Commit: ab2efce8fb4015523a9a818f0e03fa57c6366585
Parents: 2a69bcc
Author: Hyunsik Choi <hyunsik@apache.org>
Authored: Fri Dec 5 11:43:15 2014 +0900
Committer: Hyunsik Choi <hyunsik@apache.org>
Committed: Fri Dec 5 16:36:12 2014 +0900

----------------------------------------------------------------------
 CHANGES                                         |  7 ++--
 .../org/apache/tajo/client/TestTajoClient.java  | 37 +++++++++++++++++---
 .../org/apache/tajo/plan/LogicalPlanner.java    | 20 ++++++++---
 .../org/apache/tajo/plan/util/PlannerUtil.java  | 17 +++++++++
 4 files changed, 70 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/ab2efce8/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 0c785ce..8ecd795 100644
--- a/CHANGES
+++ b/CHANGES
@@ -83,8 +83,11 @@ Release 0.9.1 - unreleased
 
   BUG FIXES
 
-    TAJO-1194: 'INSERT OVERWRITE .. SELECT' does not remove existing data when result is
empty.
-    (jaehwa)
+    TAJO-1231: Implicit table properties in session are not stored in 
+    table property. (hyunsik)
+
+    TAJO-1194: 'INSERT OVERWRITE .. SELECT' does not remove existing data 
+    when result is empty. (jaehwa)
 
     TAJO-1191: Change DateDatum timezone to UTC. (Jaewoong Jung via hyunsik)
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/ab2efce8/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
index 616799f..21dd7d5 100644
--- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
+++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
@@ -680,7 +680,7 @@ public class TestTajoClient {
   }
 
   @Test
-  public void testSetCvsNull() throws Exception {
+  public void testNullCharSession() throws Exception {
     String sql =
         "select\n" +
             "  c_custkey,\n" +
@@ -692,17 +692,44 @@ public class TestTajoClient {
             "  c_custkey,\n" +
             "  orders.o_orderkey;\n";
 
-    TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
-
     Map<String, String> variables = new HashMap<String, String>();
     variables.put(SessionVars.NULL_CHAR.keyname(), "\\\\T");
     client.updateSessionVariables(variables);
+    TajoResultSet resultDesc = (TajoResultSet)client.executeQueryAndGetResult(sql);
+    resultDesc.close();
+    assertNullCharSessionVar(resultDesc.getTableDesc());
+  }
+
+  @Test
+  public void testNullCharSessionInCTAS() throws Exception {
+    String sql =
+        "create table nullcharsession as select\n" +
+            "  c_custkey,\n" +
+            "  orders.o_orderkey,\n" +
+            "  orders.o_orderstatus \n" +
+            "from\n" +
+            "  orders full outer join customer on c_custkey = o_orderkey\n" +
+            "order by\n" +
+            "  c_custkey,\n" +
+            "  orders.o_orderkey;\n";
 
+    Map<String, String> variables = new HashMap<String, String>();
+    variables.put(SessionVars.NULL_CHAR.keyname(), "\\\\T");
+    client.updateSessionVariables(variables);
     TajoResultSet res = (TajoResultSet)client.executeQueryAndGetResult(sql);
+    res.close();
+
+    TableDesc resultDesc = client.getTableDesc("nullcharsession");
+    assertNullCharSessionVar(resultDesc);
+  }
+
+
+  public void assertNullCharSessionVar(TableDesc resultDesc) throws Exception {
+    TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
 
-    assertEquals(res.getTableDesc().getMeta().getOption(StorageConstants.TEXT_NULL), "\\\\T");
+    assertEquals(resultDesc.getMeta().getOption(StorageConstants.TEXT_NULL), "\\\\T");
 
-    Path path = new Path(res.getTableDesc().getPath());
+    Path path = new Path(resultDesc.getPath());
     FileSystem fs = path.getFileSystem(tajoConf);
 
     FileStatus[] files = fs.listStatus(path);

http://git-wip-us.apache.org/repos/asf/tajo/blob/ab2efce8/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index 3b1b4e3..69c0e4b 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -1706,13 +1706,25 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
       createTableNode.setStorageType(CatalogProtos.StoreType.CSV);
     }
 
-    // Set default storage properties to be created.
-    KeyValueSet keyValueSet = CatalogUtil.newPhysicalProperties(createTableNode.getStorageType());
+
+
+    // Set default storage properties to table
+    KeyValueSet properties = CatalogUtil.newPhysicalProperties(createTableNode.getStorageType());
+
+    // Priority to apply table properties
+    // 1. Explicit table properties specified in WITH clause
+    // 2. Session variables
+
+    // Set session variables to properties
+    PlannerUtil.applySessionToTableProperties(context.queryContext, createTableNode.getStorageType(),
properties);
+    // Set table properties specified in WITH clause
     if (expr.hasParams()) {
-      keyValueSet.putAll(expr.getParams());
+      properties.putAll(expr.getParams());
     }
 
-    createTableNode.setOptions(keyValueSet);
+    createTableNode.setOptions(properties);
+
+
 
     if (expr.hasPartition()) {
       if (expr.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) {

http://git-wip-us.apache.org/repos/asf/tajo/blob/ab2efce8/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
index 6868b6c..c55c203 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
@@ -21,6 +21,8 @@ package org.apache.tajo.plan.util;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
+import org.apache.tajo.OverridableConf;
+import org.apache.tajo.SessionVars;
 import org.apache.tajo.algebra.*;
 import org.apache.tajo.annotation.Nullable;
 import org.apache.tajo.catalog.Column;
@@ -35,10 +37,15 @@ import org.apache.tajo.plan.logical.*;
 import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
 import org.apache.tajo.plan.visitor.ExplainLogicalPlanVisitor;
 import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor;
+import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.util.KeyValueSet;
 import org.apache.tajo.util.TUtil;
 
 import java.util.*;
 
+import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType.CSV;
+import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType.TEXTFILE;
+
 public class PlannerUtil {
 
   public static boolean checkIfDDLPlan(LogicalNode node) {
@@ -776,4 +783,14 @@ public class PlannerUtil {
 
     return explains.toString();
   }
+
+  public static void applySessionToTableProperties(OverridableConf sessionVars,
+                                                   CatalogProtos.StoreType storeType,
+                                                   KeyValueSet tableProperties) {
+    if (storeType == CSV || storeType == TEXTFILE) {
+      if (sessionVars.containsKey(SessionVars.NULL_CHAR)) {
+        tableProperties.set(StorageConstants.TEXT_NULL, sessionVars.get(SessionVars.NULL_CHAR));
+      }
+    }
+  }
 }


Mime
View raw message