tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jihoon...@apache.org
Subject git commit: TAJO-59: Implement Char Datum Type. (jihoon)
Date Thu, 29 Aug 2013 01:38:14 GMT
Updated Branches:
  refs/heads/master a09ac15ca -> 4e47471c3


TAJO-59: Implement Char Datum Type. (jihoon)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/4e47471c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/4e47471c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/4e47471c

Branch: refs/heads/master
Commit: 4e47471c3c72756821c26c6817611b6cc51c71b4
Parents: a09ac15
Author: Jihoon Son <jihoonson@apache.org>
Authored: Thu Aug 29 10:37:23 2013 +0900
Committer: Jihoon Son <jihoonson@apache.org>
Committed: Thu Aug 29 10:37:23 2013 +0900

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 +
 .../org/apache/tajo/catalog/CatalogUtil.java    |  4 +
 .../java/org/apache/tajo/catalog/Column.java    |  4 +
 .../java/org/apache/tajo/catalog/Schema.java    |  7 ++
 .../org/apache/tajo/catalog/store/DBStore.java  | 16 ++--
 .../java/org/apache/tajo/datum/CharDatum.java   | 84 ++++++++++++--------
 .../org/apache/tajo/datum/DatumFactory.java     | 14 ++--
 .../org/apache/tajo/datum/TestCharDatum.java    | 26 +++---
 .../main/java/org/apache/tajo/cli/TajoCli.java  |  3 +
 .../tajo/engine/planner/LogicalPlanner.java     | 37 ++++++++-
 .../java/org/apache/tajo/storage/CSVFile.java   | 11 +--
 .../java/org/apache/tajo/storage/RawFile.java   | 12 ++-
 .../java/org/apache/tajo/storage/RowFile.java   | 38 +++------
 .../tajo/storage/rcfile/RCFileWrapper.java      |  3 +-
 .../tajo/storage/trevni/TrevniAppender.java     |  4 +-
 .../tajo/storage/trevni/TrevniScanner.java      |  3 +-
 .../org/apache/tajo/storage/TestStorages.java   | 13 +--
 17 files changed, 178 insertions(+), 103 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 03dfaf3..787c0b5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,8 @@ Tajo Change Log
 Release 0.2.0 - unreleased
 
   NEW FEATURES
+
+    TAJO-59: Implement Char Datum Type. (jihoon)
    
     TAJO-96: Design and implement rewrite rule interface and the rewrite rule 
     engine. (hyunsik)

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 3eb08da..8f17852 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -178,4 +178,8 @@ public class CatalogUtil {
     }
     return dataTypes;
   }
+
+  public static DataType newDataTypeWithLen(Type type, int length) {
+    return DataType.newBuilder().setType(type).setLength(length).build();
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
index c215661..2227999 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
@@ -45,6 +45,10 @@ public class Column implements ProtoObject<ColumnProto>, Cloneable,
GsonObject {
   public Column(String columnName, TajoDataTypes.Type type) {
     this(columnName, CatalogUtil.newDataTypeWithoutLen(type));
   }
+
+  public Column(String columnName, TajoDataTypes.Type type, int typeLength) {
+    this(columnName, CatalogUtil.newDataTypeWithLen(type, typeLength));
+  }
 	
 	public Column(ColumnProto proto) {
 		this(proto.getColumnName(), proto.getDataType());

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
index 9fee40f..65f6176 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
@@ -114,9 +114,16 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable,
GsonObject {
 	}
 
   public synchronized Schema addColumn(String name, Type type) {
+    if (type == Type.CHAR) {
+      return addColumn(name, CatalogUtil.newDataTypeWithLen(type, 1));
+    }
     return addColumn(name, CatalogUtil.newDataTypeWithoutLen(type));
   }
 
+  public synchronized Schema addColumn(String name, Type type, int length) {
+    return addColumn(name, CatalogUtil.newDataTypeWithLen(type, length));
+  }
+
   public synchronized Schema addColumn(String name, DataType dataType) {
 		String lowcased = name.toLowerCase();
 		if(fieldsByName.containsKey(lowcased)) {

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DBStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DBStore.java
b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DBStore.java
index 8fb19f0..078ca6e 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DBStore.java
@@ -216,7 +216,7 @@ public class DBStore implements CatalogStore {
           + C_TABLE_ID + " VARCHAR(256) NOT NULL REFERENCES " + TB_TABLES + "("
           + C_TABLE_ID + ") ON DELETE CASCADE, "
           + "column_id INT NOT NULL,"
-          + "column_name VARCHAR(256) NOT NULL, " + "data_type CHAR(16), "
+          + "column_name VARCHAR(256) NOT NULL, " + "data_type CHAR(16), " + "type_length
INTEGER, "
           + "CONSTRAINT C_COLUMN_ID UNIQUE (" + C_TABLE_ID + ", column_name))";
       if (LOG.isDebugEnabled()) {
         LOG.debug(columns_ddl);
@@ -439,13 +439,14 @@ public class DBStore implements CatalogStore {
       final int columnId, final Column col) {
     String sql =
         "INSERT INTO " + TB_COLUMNS 
-        + " (tid, " + C_TABLE_ID + ", column_id, column_name, data_type) "
+        + " (tid, " + C_TABLE_ID + ", column_id, column_name, data_type, type_length) "
         + "VALUES("
         + tid + ","
         + "'" + desc.getName() + "',"
         + columnId + ", "
         + "'" + col.getColumnName() + "',"
-        + "'" + col.getDataType().getType().name() + "'"
+        + "'" + col.getDataType().getType().name() + "',"
+        + (col.getDataType().hasLength() ? col.getDataType().getLength() : 0)
         + ")";
     
     return sql;
@@ -615,7 +616,7 @@ public class DBStore implements CatalogStore {
       
       Schema schema = null;
       try {
-        String sql = "SELECT column_name, data_type from " + TB_COLUMNS
+        String sql = "SELECT column_name, data_type, type_length from " + TB_COLUMNS
             + " WHERE " + C_TABLE_ID + "='" + name + "' ORDER by column_id asc";
 
         stmt = conn.createStatement();
@@ -630,7 +631,12 @@ public class DBStore implements CatalogStore {
               + res.getString("column_name").trim();
           Type dataType = getDataType(res.getString("data_type")
               .trim());
-          schema.addColumn(columnName, dataType);
+          int typeLength = res.getInt("type_length");
+          if (typeLength > 0) {
+            schema.addColumn(columnName, dataType, typeLength);
+          } else {
+            schema.addColumn(columnName, dataType);
+          }
         }
       } catch (SQLException se) {
         throw new IOException(se);

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java
index 4d77dad..9ce9089 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java
@@ -21,72 +21,96 @@ package org.apache.tajo.datum;
 import com.google.gson.annotations.Expose;
 import org.apache.tajo.datum.exception.InvalidOperationException;
 
+import java.util.Arrays;
+
 import static org.apache.tajo.common.TajoDataTypes.Type;
 
 public class CharDatum extends Datum {
-  private static final int size = 1;
-  @Expose char val;
+  @Expose private int size;
+  @Expose private byte[] bytes;
+  private String chars = null;
 
 	public CharDatum() {
 		super(Type.CHAR);
 	}
 
 	public CharDatum(byte val) {
-		this();
-		this.val = (char)val;
+    this();
+    this.size = 1;
+    bytes = new byte[size];
+    bytes[0] = val;
 	}
 
+  public CharDatum(char val) {
+    this((byte)val);
+  }
+
   public CharDatum(byte [] bytes) {
-    this(bytes[0]);
+    this();
+    this.bytes = bytes;
+    this.size = bytes.length;
   }
 
-	public CharDatum(char val) {
-	  this();
-	  this.val = val;
-	}
+  public CharDatum(String val) {
+    this(val.getBytes());
+  }
+
+  private String getString() {
+    if (chars == null) {
+      chars = new String(bytes);
+    }
+    return chars;
+  }
 
   @Override
   public char asChar() {
-    return val;
+    return getString().charAt(0);
   }
 
   @Override
-	public int asInt4() {
-		return val;
-	}
+  public short asInt2() {
+    return Short.valueOf(getString());
+  }
 
   @Override
-	public long asInt8() {
-		return val;
+	public int asInt4() {
+		return Integer.valueOf(getString());
 	}
 
   @Override
-	public byte asByte() {
-		return (byte)val;
+	public long asInt8() {
+		return Long.valueOf(getString());
 	}
 
   @Override
 	public byte[] asByteArray() {
-		byte [] bytes = new byte[1];
-    bytes[0] = (byte) val;
 		return bytes;
 	}
 
   @Override
 	public float asFloat4() {
-		return val;
+    return Float.valueOf(getString());
 	}
 
   @Override
 	public double asFloat8() {
-		return val;
+		return Double.valueOf(getString());
 	}
 
   @Override
+  public byte asByte() {
+    return bytes[0];
+  }
+
+  @Override
 	public String asChars() {
-		return String.valueOf(val);
+		return getString();
 	}
 
+  /**
+   * Return the real length of the string
+   * @return the length of the string
+   */
   @Override
   public int size() {
     return size;
@@ -94,14 +118,15 @@ public class CharDatum extends Datum {
   
   @Override
   public int hashCode() {
-    return val;
+    return getString().hashCode();
   }
 
   @Override
   public boolean equals(Object obj) {
     if (obj instanceof CharDatum) {
       CharDatum other = (CharDatum) obj;
-      return val == other.val;
+      return this.size == other.size &&
+          Arrays.equals(this.bytes, other.bytes);
     }
     
     return false;
@@ -111,7 +136,7 @@ public class CharDatum extends Datum {
   public BooleanDatum equalsTo(Datum datum) {
     switch (datum.type()) {
     case CHAR:
-      return DatumFactory.createBool(this.val == (((CharDatum) datum).val));
+      return DatumFactory.createBool(this.equals(datum));
     default:
       throw new InvalidOperationException(datum.type());
     }
@@ -121,13 +146,8 @@ public class CharDatum extends Datum {
   public int compareTo(Datum datum) {
     switch (datum.type()) {
       case CHAR:
-      if (val < datum.asChar()) {
-        return -1;
-      } else if (val > datum.asChar()) {
-        return 1;
-      } else {
-        return 0;
-      }
+        CharDatum other = (CharDatum) datum;
+        return this.getString().compareTo(other.getString());
     default:
       throw new InvalidOperationException(datum.type());
     }

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
index 453dd4a..72dfc15 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
@@ -23,6 +23,8 @@ import org.apache.tajo.common.TajoDataTypes.DataType;
 import org.apache.tajo.common.TajoDataTypes.Type;
 import org.apache.tajo.util.Bytes;
 
+import java.io.UnsupportedEncodingException;
+
 public class DatumFactory {
 
   public static Datum create(DataType type, byte[] val) {
@@ -78,7 +80,7 @@ public class DatumFactory {
       case FLOAT8:
         return createFloat8(Bytes.toDouble(val));
       case CHAR:
-        return createChar(val[0]);
+        return createChar(val);
       case TEXT:
         return createText(val);
       case BIT:
@@ -131,12 +133,14 @@ public class DatumFactory {
     return new CharDatum(val);
   }
 
-  /*
-  public static CharDatum createChar(Integer val) {
+  public static CharDatum createChar(byte[] bytes) {
+    return new CharDatum(bytes);
+  }
+
+  public static CharDatum createChar(String val) {
     return new CharDatum(val);
   }
-  */
-	
+
 	public static Int2Datum createInt2(short val) {
 		return new Int2Datum(val);
 	}

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-common/src/test/java/org/apache/tajo/datum/TestCharDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestCharDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestCharDatum.java
index 3e6ed10..1d6da28 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestCharDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestCharDatum.java
@@ -34,51 +34,53 @@ public class TestCharDatum {
 
 	@Test
 	public final void testAsInt() {
-		Datum d = DatumFactory.createChar((char)5);
+		Datum d = DatumFactory.createChar("5");
 		assertEquals(5,d.asInt4());
 	}
 	
 	@Test
 	public final void testAsLong() {
-		Datum d = DatumFactory.createChar((char)5);
+		Datum d = DatumFactory.createChar("5");
 		assertEquals(5l,d.asInt8());
 	}
 	
 	@Test
 	public final void testAsByte() {
 		Datum d = DatumFactory.createChar((char)5);
-		assertEquals(5,d.asInt8());
+		assertEquals(5,d.asByte());
 	}
 
 	@Test
 	public final void testAsFloat() {
-		Datum d = DatumFactory.createChar((char)5);
+		Datum d = DatumFactory.createChar("5");
 		assertTrue(5.0f == d.asFloat4());
 	}
 
 	@Test
 	public final void testAsDouble() {
-		Datum d = DatumFactory.createChar((char)5);
+		Datum d = DatumFactory.createChar("5");
 		assertTrue(5.0d == d.asFloat8());
 	}
 	
 	@Test
 	public final void testAsChars() {
-		Datum d = DatumFactory.createChar((char)5);
-		System.out.println(d.asChars());
+		Datum d = DatumFactory.createChar("1234567890");
+    assertEquals("1234567890", d.asChars());
 	}
 	
 	@Test
   public final void testSize() {
-    Datum d = DatumFactory.createChar((char) 1);
-    assertEquals(1, d.size());
+    Datum d = DatumFactory.createChar("1234567890".getBytes());
+    assertEquals(10, d.size());
+    d = DatumFactory.createChar("1234567890");
+    assertEquals(10, d.size());
   }
 
   @Test
   public final void testCompare() {
-    Datum a = DatumFactory.createChar('a');
-    Datum b = DatumFactory.createChar('b');
-    Datum c = DatumFactory.createChar('c');
+    Datum a = DatumFactory.createChar("abc");
+    Datum b = DatumFactory.createChar("abd");
+    Datum c = DatumFactory.createChar("ace");
 
     assertTrue(a.compareTo(b) < 0);
     assertTrue(b.compareTo(a) > 0);

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java
index 55a97ca..083328e 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java
@@ -441,6 +441,9 @@ public class TajoCli {
     for(int i = 0; i < desc.getMeta().getSchema().getColumnNum(); i++) {
       Column col = desc.getMeta().getSchema().getColumn(i);
       sb.append(col.getColumnName()).append("\t").append(col.getDataType().getType());
+      if (col.getDataType().hasLength()) {
+        sb.append("(").append(col.getDataType().getLength()).append(")");
+      }
       sb.append("\n");
     }
     return sb.toString();

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 4fa4a10..2c4f241 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -24,6 +24,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
 import org.apache.tajo.algebra.*;
+import org.apache.tajo.algebra.CreateTable.ColumnDefinition;
 import org.apache.tajo.catalog.*;
 import org.apache.tajo.catalog.function.AggFunction;
 import org.apache.tajo.catalog.function.GeneralFunction;
@@ -785,16 +786,44 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
   private Schema convertTableElementsSchema(CreateTable.ColumnDefinition [] elements) {
     Schema schema = new Schema();
 
-    Column column;
     for (CreateTable.ColumnDefinition columnDefinition: elements) {
-      column = new Column(columnDefinition.getColumnName(),
-          TajoDataTypes.Type.valueOf(columnDefinition.getDataType()));
-      schema.addColumn(column);
+      schema.addColumn(convertColumn(columnDefinition));
     }
 
     return schema;
   }
 
+  private Column convertColumn(ColumnDefinition columnDefinition) {
+    TajoDataTypes.Type type = TajoDataTypes.Type.valueOf(columnDefinition.getDataType());
+    Column column;
+    switch (type) {
+      case CHAR:
+      case VARCHAR:
+      case NCHAR:
+      case NVARCHAR:
+        column = new Column(columnDefinition.getColumnName(),
+            TajoDataTypes.Type.valueOf(columnDefinition.getDataType()),
+            columnDefinition.getLengthOrPrecision());
+        break;
+      case FLOAT4:
+      case FLOAT8:
+        // TODO: support precision
+        column = new Column(columnDefinition.getColumnName(),
+            TajoDataTypes.Type.valueOf(columnDefinition.getDataType()));
+        break;
+      case NUMERIC:
+      case DECIMAL:
+        // TODO: support precision and scale
+        column = new Column(columnDefinition.getColumnName(),
+            TajoDataTypes.Type.valueOf(columnDefinition.getDataType()));
+        break;
+      default:
+        column = new Column(columnDefinition.getColumnName(),
+            TajoDataTypes.Type.valueOf(columnDefinition.getDataType()));
+    }
+    return column;
+  }
+
   @Override
   public LogicalNode visitDropTable(PlanContext context, Stack<OpType> stack, DropTable
dropTable) {
     DropTableNode dropTableNode = new DropTableNode(dropTable.getTableName());

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
index 0896bd1..5c4fa86 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
@@ -106,11 +106,11 @@ public class CSVFile {
                   .asByteArray(), false)));
               break;
             case CHAR:
-              sb.append(tuple.getChar(i));
+              CharDatum charDatum = tuple.getChar(i);
+              sb.append(charDatum);
+              byte[] pad = new byte[col.getDataType().getLength()-charDatum.size()];
+              sb.append(new String(pad));
               break;
-//          case STRING:
-//            sb.append(tuple.getString(i));
-//            break;
             case TEXT:
               TextDatum td = tuple.getText(i);
               sb.append(td.toString());
@@ -361,7 +361,8 @@ public class CSVFile {
                   tuple.put(tid, DatumFactory.createBit(Base64.decodeBase64(cell)[0]));
                   break;
                 case CHAR:
-                  tuple.put(tid, DatumFactory.createChar(cell.charAt(0)));
+                  String trimmed = cell.trim();
+                  tuple.put(tid, DatumFactory.createChar(trimmed));
                   break;
                 case BLOB:
                   tuple.put(tid, DatumFactory.createBlob(Base64.decodeBase64(cell)));

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
index c8127b1..7c0ea87 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
@@ -35,6 +35,7 @@ import java.io.RandomAccessFile;
 import java.net.URI;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
+import java.util.Arrays;
 
 public class RawFile {
   public static class RawFileScanner extends FileScanner implements SeekableScanner {
@@ -153,7 +154,11 @@ public class RawFile {
             break;
 
           case CHAR :
-            tuple.put(i, DatumFactory.createChar(buffer.getChar()));
+            int realLen = buffer.getInt();
+            byte[] buf = new byte[columnTypes[i].getLength()];
+            buffer.get(buf);
+            byte[] charBuf = Arrays.copyOf(buf, realLen);
+            tuple.put(i, DatumFactory.createChar(charBuf));
             break;
 
           case INT2 :
@@ -363,7 +368,10 @@ public class RawFile {
             break;
 
           case CHAR :
-            buffer.putChar(t.get(i).asChar());
+            byte[] src = t.getChar(i).asByteArray();
+            byte[] dst = Arrays.copyOf(src, columnTypes[i].getLength());
+            buffer.putInt(src.length);
+            buffer.put(dst);
             break;
 
           case INT2 :

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java
index 6a3bcf1..0911102 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java
@@ -211,8 +211,11 @@ public class RowFile {
               break;
 
             case CHAR :
-              datum = DatumFactory.createChar(buffer.getChar());
-              tuple.put(i, datum);
+              int realLen = buffer.getInt();
+              byte[] buf = new byte[col.getDataType().getLength()];
+              buffer.get(buf);
+              byte[] charBuf = Arrays.copyOf(buf, realLen);
+              tuple.put(i, DatumFactory.createChar(charBuf));
               break;
 
             case INT2 :
@@ -401,74 +404,51 @@ public class RowFile {
           col = schema.getColumn(i);
           switch (col.getDataType().getType()) {
             case BOOLEAN:
-//              out.writeBoolean(t.getByte(i).asBool());
               buffer.put(t.getBoolean(i).asByte());
               break;
             case BIT:
-//              out.writeByte(t.getByte(i).asByte());
               buffer.put(t.getByte(i).asByte());
               break;
             case CHAR:
-//              out.writeChar(t.getChar(i).asChar());
-              buffer.putChar(t.getChar(i).asChar());
+              byte[] src = t.getChar(i).asByteArray();
+              byte[] dst = Arrays.copyOf(src, col.getDataType().getLength());
+              buffer.putInt(src.length);
+              buffer.put(dst);
               break;
-//            case TEXT:
-//              byte[] buf = t.getString(i).asByteArray();
-//              if (buf.length > 256) {
-//                buf = new byte[256];
-//                byte[] str = t.getString(i).asByteArray();
-//                System.arraycopy(str, 0, buf, 0, 256);
-//              }
-////              out.writeShort(buf.length);
-////              out.write(buf, 0, buf.length);
-//              buffer.putShort((short)buf.length);
-//              buffer.put(buf, 0, buf.length);
-//              break;
             case TEXT:
               byte [] strbytes = t.getText(i).asByteArray();
               buffer.putShort((short)strbytes.length);
               buffer.put(strbytes, 0, strbytes.length);
               break;
             case INT2:
-//              out.writeShort(t.getShort(i).asShort());
               buffer.putShort(t.getShort(i).asInt2());
               break;
             case INT4:
-//              out.writeInt(t.getInt(i).asInt());
               buffer.putInt(t.getInt(i).asInt4());
               break;
             case INT8:
-//              out.writeLong(t.getLong(i).asLong());
               buffer.putLong(t.getLong(i).asInt8());
               break;
             case FLOAT4:
-//              out.writeFloat(t.getFloat(i).asFloat());
               buffer.putFloat(t.getFloat(i).asFloat4());
               break;
             case FLOAT8:
-//              out.writeDouble(t.getDouble(i).asDouble());
               buffer.putDouble(t.getDouble(i).asFloat8());
               break;
             case BLOB:
               byte [] bytes = t.getBytes(i).asByteArray();
-//              out.writeInt(bytes.length);
-//              out.write(bytes);
               buffer.putShort((short)bytes.length);
               buffer.put(bytes);
               break;
             case INET4:
-//              out.write(t.getIPv4Bytes(i));
               buffer.put(t.getIPv4Bytes(i));
               break;
             case INET6:
-//              out.write(t.getIPv6Bytes(i));
               buffer.put(t.getIPv6Bytes(i));
             case ARRAY: {
               ArrayDatum array = (ArrayDatum) t.get(i);
               String json = array.toJson();
               byte [] byteArray = json.getBytes();
-//              out.writeInt(byteArray.length);
-//              out.write(byteArray);
               buffer.putShort((short)byteArray.length);
               buffer.put(byteArray);
               break;

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java
index 92d9acc..6332416 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java
@@ -250,8 +250,9 @@ public class RCFileWrapper {
                   DatumFactory.createBit(column.get(tid).getBytesCopy()[0]));
               break;
             case CHAR:
+              byte[] buf = column.get(tid).getBytesCopy();
               tuple.put(tid,
-                  DatumFactory.createChar(column.get(tid).getBytesCopy()[0]));
+                  DatumFactory.createChar(buf));
               break;
 
             case INT2:

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java
index 11dcfbf..eef95e0 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java
@@ -89,7 +89,7 @@ public class TrevniAppender extends FileAppender {
       case BIT:
         return ValueType.INT;
       case CHAR:
-        return ValueType.INT;
+        return ValueType.STRING;
       case INT2:
         return ValueType.INT;
       case INT4:
@@ -134,7 +134,6 @@ public class TrevniAppender extends FileAppender {
         switch (col.getDataType().getType()) {
           case BOOLEAN:
           case BIT:
-          case CHAR:
           case INT2:
           case INT4:
             writer.writeValue(t.get(i).asInt4(), i);
@@ -148,6 +147,7 @@ public class TrevniAppender extends FileAppender {
           case FLOAT8:
             writer.writeValue(t.get(i).asFloat8(), i);
             break;
+          case CHAR:
           case TEXT:
             writer.writeValue(t.get(i).asChars(), i);
             break;

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java
index ef43671..be23d48 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java
@@ -93,8 +93,9 @@ public class TrevniScanner extends FileScanner {
               DatumFactory.createBit(((Integer) columns[i].nextValue()).byteValue()));
           break;
         case CHAR:
+          String str = (String) columns[i].nextValue();
           tuple.put(tid,
-              DatumFactory.createChar(((Integer)columns[i].nextValue()).byteValue()));
+              DatumFactory.createChar(str));
           break;
 
         case INT2:

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
index 0bc2a80..22207fe 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -185,7 +185,7 @@ public class TestStorages {
     Schema schema = new Schema();
     schema.addColumn("col1", Type.BOOLEAN);
     schema.addColumn("col2", Type.BIT);
-    schema.addColumn("col3", Type.CHAR);
+    schema.addColumn("col3", Type.CHAR, 7);
     schema.addColumn("col4", Type.INT2);
     schema.addColumn("col5", Type.INT4);
     schema.addColumn("col6", Type.INT8);
@@ -206,7 +206,7 @@ public class TestStorages {
     tuple.put(new Datum[] {
         DatumFactory.createBool(true),
         DatumFactory.createBit((byte) 0x99),
-        DatumFactory.createChar('7'),
+        DatumFactory.createChar("hyunsik"),
         DatumFactory.createInt2((short) 17),
         DatumFactory.createInt4(59),
         DatumFactory.createInt8(23l),
@@ -225,9 +225,12 @@ public class TestStorages {
     Fragment fragment = new Fragment("table", tablePath, meta, 0, status.getLen());
     Scanner scanner =  StorageManager.getScanner(conf, meta, fragment);
     scanner.init();
-    Tuple retrieved = scanner.next();
-    for (int i = 0; i < tuple.size(); i++) {
-      assertEquals(tuple.get(i), retrieved.get(i));
+
+    Tuple retrieved;
+    while ((retrieved=scanner.next()) != null) {
+      for (int i = 0; i < tuple.size(); i++) {
+        assertEquals(tuple.get(i), retrieved.get(i));
+      }
     }
   }
 }


Mime
View raw message