tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject git commit: TAJO-270: Boolean datum compatible to apache hive. (jinho)
Date Tue, 22 Oct 2013 07:29:13 GMT
Updated Branches:
  refs/heads/master b55918bd1 -> 85cff2e71


TAJO-270: Boolean datum compatible to apache hive. (jinho)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/85cff2e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/85cff2e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/85cff2e7

Branch: refs/heads/master
Commit: 85cff2e7101e6e059285a80268ba2d8aaaca00ec
Parents: b55918b
Author: jinossy <jinossy@gmail.com>
Authored: Tue Oct 22 16:27:49 2013 +0900
Committer: jinossy <jinossy@gmail.com>
Committed: Tue Oct 22 16:27:49 2013 +0900

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 +
 .../org/apache/tajo/datum/BooleanDatum.java     |  2 +-
 .../org/apache/tajo/datum/DatumFactory.java     |  7 +---
 .../tajo/engine/planner/LogicalPlanner.java     |  3 +-
 .../apache/tajo/master/querymaster/Query.java   |  4 +-
 .../java/org/apache/tajo/storage/CSVFile.java   | 44 +++++++++++++++-----
 .../java/org/apache/tajo/storage/LazyTuple.java |  2 +-
 7 files changed, 44 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/85cff2e7/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index a088588..9bb67fe 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -59,6 +59,8 @@ Release 0.2.0 - unreleased
 
   IMPROVEMENTS
 
+    TAJO-270: Boolean datum compatible to apache hive. (jinho)
+
     TAJO-261: Rearrange default port numbers and config names. (hyunsik)
 
     TAJO-236: Implement LogicalPlanVerifier to check if a logical plan is 

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/85cff2e7/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java
index aeab5aa..bf4d83d 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java
@@ -48,7 +48,7 @@ public class BooleanDatum extends Datum {
   public BooleanDatum(byte[] bytes) {
     this(bytes[0]);
   }
-	
+
 	public boolean asBool() {
 		return val;
 	}

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/85cff2e7/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
index e783666..5268ca5 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
@@ -94,12 +94,7 @@ public class DatumFactory {
   public static NullDatum createNullDatum() {
     return NullDatum.get();
   }
-  
-  public static BooleanDatum createBool(String val) {
-    boolean boolVal = val.equalsIgnoreCase("t");
-    return new BooleanDatum(boolVal);
-  }
-  
+
   public static BooleanDatum createBool(byte val) {
     boolean boolVal = val == 0x01;
     return new BooleanDatum(boolVal);

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/85cff2e7/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 9f7fe25..84cbe14 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -915,7 +915,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
         LiteralValue literal = (LiteralValue) expr;
         switch (literal.getValueType()) {
           case Boolean:
-            return new ConstEval(DatumFactory.createBool(literal.getValue()));
+            char val = literal.getValue().charAt(0);
+            return new ConstEval(DatumFactory.createBool(val == 't' || val == 'T'));
           case String:
             return new ConstEval(DatumFactory.createText(literal.getValue()));
           case Unsigned_Integer:

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/85cff2e7/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
index 715f00c..ce945b4 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.Clock;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.state.*;
+import org.apache.tajo.catalog.statistics.TableStat;
 import org.apache.tajo.engine.planner.global.DataChannel;
 import org.apache.tajo.ExecutionBlockId;
 import org.apache.tajo.QueryId;
@@ -382,9 +383,10 @@ public class Query implements EventHandler<QueryEvent> {
       try {
         FileSystem fs = finalOutputDir.getFileSystem(query.systemConf);
         ContentSummary directorySummary = fs.getContentSummary(finalOutputDir);
+        if(meta.getStat() == null) meta.setStat(new TableStat());
         meta.getStat().setNumBytes(directorySummary.getLength());
       } catch (IOException e) {
-        LOG.error(e);
+        LOG.error(e.getMessage(), e);
       }
       TableDesc outputTableDesc = new TableDescImpl(outputTableName, meta, finalOutputDir);
       TableDesc finalTableDesc = outputTableDesc;

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/85cff2e7/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
index 58d772e..550148c 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
@@ -47,11 +47,15 @@ import java.io.*;
 import java.util.Arrays;
 
 public class CSVFile {
+  public static byte[] trueBytes = "true".getBytes();
+  public static byte[] falseBytes = "false".getBytes();
+
   public static final String DELIMITER = "csvfile.delimiter";
   public static final String NULL = "csvfile.null";     //read only
   public static final String DELIMITER_DEFAULT = "|";
   public static final byte LF = '\n';
   public static int EOF = -1;
+
   private static final Log LOG = LogFactory.getLog(CSVFile.class);
 
   public static class CSVAppender extends FileAppender {
@@ -135,26 +139,46 @@ public class CSVFile {
         LazyTuple  lTuple = (LazyTuple)tuple;
         for (int i = 0; i < colNum; i++) {
           TajoDataTypes.DataType dataType = schema.getColumn(i).getDataType();
-          datum = tuple.get(i);
 
           switch (dataType.getType()) {
-            case TEXT:
-              outputStream.write(datum.asTextBytes());
+            case TEXT: {
+              datum = tuple.get(i);
+              if (datum instanceof NullDatum) {
+                outputStream.write(nullChars);
+              } else {
+                outputStream.write(datum.asTextBytes());
+              }
+              break;
+            }
+            case CHAR: {
+              datum = tuple.get(i);
+              if (datum instanceof NullDatum) {
+                outputStream.write(nullChars);
+              } else {
+                byte[] pad = new byte[dataType.getLength() - datum.size()];
+                outputStream.write(datum.asTextBytes());
+                outputStream.write(pad);
+              }
               break;
-            case CHAR:
-              byte[] pad = new byte[dataType.getLength() - datum.size()];
-              outputStream.write(datum.asTextBytes());
-              outputStream.write(pad);
+            }
+            case BOOLEAN: {
+              datum = tuple.get(i);
+              if (datum instanceof NullDatum) {
+                //null datum is zero length byte array
+              } else {
+                outputStream.write(datum.asBool() ? trueBytes : falseBytes);   //Compatibility
with Apache Hive
+              }
               break;
+            }
             case NULL:
-              outputStream.write(nullChars);
               break;
             case PROTOBUF:
+              datum = tuple.get(i);
               ProtobufDatum protobufDatum = (ProtobufDatum) datum;
               protobufJsonFormat.print(protobufDatum.get(), outputStream);
               break;
             default:
-              outputStream.write(lTuple.getTextBytes(i));
+              outputStream.write(lTuple.getTextBytes(i)); //better usage for insertion to
table of lazy tuple
               break;
           }
 
@@ -179,7 +203,7 @@ public class CSVFile {
             col = schema.getColumn(i);
             switch (col.getDataType().getType()) {
               case BOOLEAN:
-                outputStream.write(tuple.getBoolean(i).asTextBytes());
+                outputStream.write(tuple.getBoolean(i).asBool() ? trueBytes : falseBytes);
  //Compatibility with Apache Hive
                 break;
               case BIT:
                 outputStream.write(tuple.getByte(i).asTextBytes());

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/85cff2e7/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
index 1d1d5fe..a395ea4 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
@@ -306,7 +306,7 @@ public class LazyTuple implements Tuple {
   private  Datum createByTextBytes(TajoDataTypes.DataType type, byte [] val) {
     switch (type.getType()) {
       case BOOLEAN:
-        return isNotNull(val) ? DatumFactory.createBool(new String(val)) : NullDatum.get();
+        return isNotNull(val) ? DatumFactory.createBool(val[0] == 't' || val[0] == 'T') :
NullDatum.get();
       case INT2:
         return isNotNull(val) ? DatumFactory.createInt2(new String(val)) : NullDatum.get();
       case INT4:


Mime
View raw message