tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject git commit: TAJO-506: RawFile cannot support DATE type. (jinho)
Date Thu, 30 Jan 2014 08:54:00 GMT
Updated Branches:
  refs/heads/master 8aa0e55d7 -> 567183fc2


TAJO-506: RawFile cannot support DATE type. (jinho)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/567183fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/567183fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/567183fc

Branch: refs/heads/master
Commit: 567183fc20e10d56390fd1b14052d082c6ef4623
Parents: 8aa0e55
Author: jinossy <jinossy@gmail.com>
Authored: Thu Jan 30 17:52:48 2014 +0900
Committer: jinossy <jinossy@gmail.com>
Committed: Thu Jan 30 17:52:48 2014 +0900

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 +
 .../org/apache/tajo/datum/TimestampDatum.java   |  5 +++
 .../apache/tajo/datum/TestTimestampDatum.java   |  5 ++-
 .../engine/planner/RangePartitionAlgorithm.java |  7 +++
 .../apache/tajo/engine/query/TestSortQuery.java | 10 +++++
 .../resources/dataset/TestSortQuery/table1.tbl  |  5 +++
 .../create_table_with_date_ddl.sql              | 10 +++++
 .../queries/TestSortQuery/testSortWithDate.sql  |  1 +
 .../TestSortQuery/testSortWithDate.result       |  7 +++
 .../java/org/apache/tajo/storage/RawFile.java   | 36 ++++++++++-----
 .../org/apache/tajo/storage/RowStoreUtil.java   |  6 ++-
 .../org/apache/tajo/storage/StorageUtil.java    | 47 ++++++++++++++------
 .../org/apache/tajo/storage/TestStorages.java   | 43 ++++++++++++++++++
 13 files changed, 156 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0fe9dc5..50b3695 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -236,6 +236,8 @@ Release 0.8.0 - unreleased
 
   BUG FIXES
 
+    TAJO-506: RawFile cannot support DATE type. (jinho)
+
     TAJO-566: BIN/TAJO_DUMP makes wrong ddl script. (hyoungjunkim via hyunsik)
 
     TAJO-567: Expression projection bugs. (hyunsik)

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
index 3ff1fca..c1fad56 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
@@ -123,6 +123,11 @@ public class TimestampDatum extends Datum {
     return dateTime.getWeekOfWeekyear();
   }
 
+  @Override
+  public long asInt8() {
+    return dateTime.getMillis();
+  }
+
   public String toString() {
     return asChars();
   }

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
index 0482857..4a6b181 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
@@ -48,10 +48,11 @@ public class TestTimestampDatum {
     d.asInt4();
 	}
 
-  @Test(expected = InvalidCastException.class)
+  @Test
 	public final void testAsInt8() {
     Datum d = DatumFactory.createTimeStamp(timestamp);
-    assertEquals(timestamp, d.asInt8());
+    long javaTime = timestamp * 1000l;
+    assertEquals(javaTime, d.asInt8());
 	}
 
   @Test(expected = InvalidCastException.class)

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java
b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java
index 35d7743..fc56b55 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java
@@ -87,6 +87,13 @@ public abstract class RangePartitionAlgorithm {
       case TEXT:
         columnCard = new BigDecimal(end.asChars().charAt(0) - start.asChars().charAt(0));
         break;
+      case DATE:
+        columnCard = new BigDecimal(end.asInt4() - start.asInt4());
+        break;
+      case TIME:
+      case TIMESTAMP:
+        columnCard = new BigDecimal(end.asInt8() - start.asInt8());
+        break;
       default:
         throw new UnsupportedOperationException(dataType + " is not supported yet");
     }

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
index 8601cb5..5e5c70c 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
@@ -93,4 +93,14 @@ public class TestSortQuery extends QueryTestCaseBase {
     assertResultSet(res);
     cleanupQuery(res);
   }
+
+  @Test
+  public final void testSortWithDate() throws Exception {
+    //select col1, col2, l_comment from table1 order by col1, col2;
+    executeDDL("create_table_with_date_ddl.sql", "table1.tbl");
+
+    ResultSet res = executeQuery();
+    assertResultSet(res);
+    cleanupQuery(res);
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-core/tajo-core-backend/src/test/resources/dataset/TestSortQuery/table1.tbl
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/dataset/TestSortQuery/table1.tbl
b/tajo-core/tajo-core-backend/src/test/resources/dataset/TestSortQuery/table1.tbl
new file mode 100644
index 0000000..a1ebde3
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/dataset/TestSortQuery/table1.tbl
@@ -0,0 +1,5 @@
+1997-11-09 20:34:56|1996-04-12|15:34:56
+1997-11-09 20:34:56|1996-03-13|19:34:56
+1993-11-09 20:34:56|1997-01-28|08:34:56
+1995-11-09 20:34:56|1994-02-02|17:34:56
+1995-11-09 20:34:56|1993-11-09|20:34:56
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/create_table_with_date_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/create_table_with_date_ddl.sql
b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/create_table_with_date_ddl.sql
new file mode 100644
index 0000000..846cbb6
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/create_table_with_date_ddl.sql
@@ -0,0 +1,10 @@
+-- Sort Table
+-- It is used in TestSortQuery::testSortWithDate
+
+create external table table1 (
+  col1 timestamp,
+	col2 date,
+	col3 time
+) using csv
+with ('csvfile.delimiter'='|', 'csvfile.null'='NULL')
+location ${table.path};
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/testSortWithDate.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/testSortWithDate.sql
b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/testSortWithDate.sql
new file mode 100644
index 0000000..490e8d0
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/testSortWithDate.sql
@@ -0,0 +1 @@
+select col1, col2, col3 from table1 order by col1, col2, col3;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-core/tajo-core-backend/src/test/resources/results/TestSortQuery/testSortWithDate.result
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestSortQuery/testSortWithDate.result
b/tajo-core/tajo-core-backend/src/test/resources/results/TestSortQuery/testSortWithDate.result
new file mode 100644
index 0000000..118909c
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestSortQuery/testSortWithDate.result
@@ -0,0 +1,7 @@
+col1,col2,col3
+-------------------------------
+1993-11-09 20:34:56,1997-01-28,08:34:56
+1995-11-09 20:34:56,1993-11-09,20:34:56
+1995-11-09 20:34:56,1994-02-02,17:34:56
+1997-11-09 20:34:56,1996-03-13,19:34:56
+1997-11-09 20:34:56,1996-04-12,15:34:56
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java
index db511dc..749b2e4 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java
@@ -30,7 +30,6 @@ import org.apache.tajo.common.TajoDataTypes.DataType;
 import org.apache.tajo.datum.DatumFactory;
 import org.apache.tajo.datum.NullDatum;
 import org.apache.tajo.datum.ProtobufDatumFactory;
-import org.apache.tajo.datum.TimestampDatum;
 import org.apache.tajo.storage.fragment.FileFragment;
 import org.apache.tajo.util.BitArray;
 
@@ -212,10 +211,6 @@ public class RawFile {
             tuple.put(i, DatumFactory.createText(new String(strBytes2)));
             break;
 
-          case TIMESTAMP:
-            tuple.put(i, DatumFactory.createTimeStampFromMillis(buffer.getLong()));
-            break;
-
           case BLOB : {
             //byte [] rawBytes = getColumnBytes();
             int byteSize = buffer.getInt();
@@ -244,6 +239,25 @@ public class RawFile {
             tuple.put(i, DatumFactory.createInet4(ipv4Bytes));
             break;
 
+          case DATE: {
+            int val = buffer.getInt();
+            if (val < Integer.MIN_VALUE + 1) {
+              tuple.put(i, DatumFactory.createNullDatum());
+            } else {
+              tuple.put(i, DatumFactory.createFromInt4(columnTypes[i], val));
+            }
+            break;
+          }
+          case TIME:
+          case TIMESTAMP: {
+            long val = buffer.getLong();
+            if (val < Long.MIN_VALUE + 1) {
+              tuple.put(i, DatumFactory.createNullDatum());
+            } else {
+              tuple.put(i, DatumFactory.createFromInt8(columnTypes[i], val));
+            }
+            break;
+          }
           case NULL_TYPE:
             tuple.put(i, NullDatum.get());
             break;
@@ -442,10 +456,6 @@ public class RawFile {
             buffer.put(strBytes2);
             break;
 
-          case TIMESTAMP:
-            buffer.putLong(((TimestampDatum)t.get(i)).getMillis());
-            break;
-
           case BLOB : {
             byte [] rawBytes = t.get(i).asByteArray();
             if (flushBufferAndReplace(recordOffset, rawBytes.length + 4)) {
@@ -480,7 +490,13 @@ public class RawFile {
           case INET4 :
             buffer.put(t.get(i).asByteArray());
             break;
-
+          case DATE:
+            buffer.putInt(t.get(i).asInt4());
+            break;
+          case TIME:
+          case TIMESTAMP:
+            buffer.putLong(t.get(i).asInt8());
+            break;
           default:
             throw new IOException("Cannot support data type: " + columnTypes[i].getType());
         }

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
index 9f32028..e54fb57 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
@@ -22,7 +22,6 @@ import org.apache.tajo.catalog.Column;
 import org.apache.tajo.catalog.Schema;
 import org.apache.tajo.common.TajoDataTypes;
 import org.apache.tajo.datum.DatumFactory;
-import org.apache.tajo.datum.TimestampDatum;
 import org.apache.tajo.util.Bytes;
 
 import java.nio.ByteBuffer;
@@ -182,7 +181,10 @@ public class RowStoreUtil {
             bb.put(_string);
             break;
           case DATE: bb.putInt(tuple.get(i).asInt4()); break;
-          case TIMESTAMP: bb.putLong(((TimestampDatum)tuple.get(i)).getMillis()); break;
+          case TIME:
+          case TIMESTAMP:
+            bb.putLong(tuple.get(i).asInt8());
+            break;
           case BLOB:
             byte [] bytes = tuple.get(i).asByteArray();
             bb.putInt(bytes.length);

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
index 9627a5d..1f13946 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
@@ -40,20 +40,39 @@ public class StorageUtil {
   }
 
   public static int getColByteSize(Column col) {
-    switch(col.getDataType().getType()) {
-    case BOOLEAN: return 1;
-    case CHAR: return 1;
-    case BIT: return 1;
-    case INT2: return 2;
-    case INT4: return 4;
-    case INT8: return 8;
-    case FLOAT4: return 4;
-    case FLOAT8: return 8;
-    case INET4: return 4;
-    case INET6: return 32;
-    case TEXT: return 256;
-    case BLOB: return 256;
-    default: return 0;
+    switch (col.getDataType().getType()) {
+      case BOOLEAN:
+        return 1;
+      case CHAR:
+        return 1;
+      case BIT:
+        return 1;
+      case INT2:
+        return 2;
+      case INT4:
+        return 4;
+      case INT8:
+        return 8;
+      case FLOAT4:
+        return 4;
+      case FLOAT8:
+        return 8;
+      case INET4:
+        return 4;
+      case INET6:
+        return 32;
+      case TEXT:
+        return 256;
+      case BLOB:
+        return 256;
+      case DATE:
+        return 4;
+      case TIME:
+        return 8;
+      case TIMESTAMP:
+        return 8;
+      default:
+        return 0;
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
index 16b370c..1685675 100644
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -245,6 +245,7 @@ public class TestStorages {
         assertEquals(tuple.get(i), retrieved.get(i));
       }
     }
+    scanner.close();
   }
 
   @Test
@@ -308,6 +309,7 @@ public class TestStorages {
         assertEquals(tuple.get(i), retrieved.get(i));
       }
     }
+    scanner.close();
   }
 
   @Test
@@ -371,5 +373,46 @@ public class TestStorages {
         assertEquals(tuple.get(i), retrieved.get(i));
       }
     }
+    scanner.close();
+  }
+
+  @Test
+  public void testTime() throws IOException {
+    if (storeType == StoreType.CSV || storeType == StoreType.RAW) {
+      Schema schema = new Schema();
+      schema.addColumn("col1", Type.DATE);
+      schema.addColumn("col2", Type.TIME);
+      schema.addColumn("col3", Type.TIMESTAMP);
+
+      Options options = new Options();
+      TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
+
+      Path tablePath = new Path(testDir, "testTime.data");
+      Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta,
schema, tablePath);
+      appender.init();
+
+      Tuple tuple = new VTuple(3);
+      tuple.put(new Datum[]{
+          DatumFactory.createDate("1980-04-01"),
+          DatumFactory.createTime("12:34:56"),
+          DatumFactory.createTimeStamp((int) System.currentTimeMillis() / 1000)
+      });
+      appender.addTuple(tuple);
+      appender.flush();
+      appender.close();
+
+      FileStatus status = fs.getFileStatus(tablePath);
+      FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
+      Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema,
fragment);
+      scanner.init();
+
+      Tuple retrieved;
+      while ((retrieved = scanner.next()) != null) {
+        for (int i = 0; i < tuple.size(); i++) {
+          assertEquals(tuple.get(i), retrieved.get(i));
+        }
+      }
+      scanner.close();
+    }
   }
 }


Mime
View raw message