arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-824: Date and Time Vectors should reflect timezone-less semantics
Date Sat, 06 May 2017 02:05:58 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 1a6d135be -> 316c63dba


ARROW-824: Date and Time Vectors should reflect timezone-less semantics

The current java vector support the timezone less version of time related types but in an
incomplete way.
This change fixes it and clarifies what the vectors implementation do.
We'll need separate vectors or adapt those to deal will timezone aware time types.

Author: Julien Le Dem <julien@apache.org>

Closes #568 from julienledem/ARROW-824 and squashes the following commits:

3528ad8 [Julien Le Dem] add license
e37385c [Julien Le Dem] centralize LocalDateTime.toMillis
bdac7ff [Julien Le Dem] make integration tests output more readable
b0da88c [Julien Le Dem] fix failing integration test
61518ec [Julien Le Dem] improve travis integration
ec19e7d [Julien Le Dem] ARROW-824: Date and Time Vectors should reflect timezone-less semantics


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/316c63db
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/316c63db
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/316c63db

Branch: refs/heads/master
Commit: 316c63dbaec8e5df33d0cf0fa78d38ac8cc375b8
Parents: 1a6d135
Author: Julien Le Dem <julien@apache.org>
Authored: Fri May 5 22:05:46 2017 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Fri May 5 22:05:46 2017 -0400

----------------------------------------------------------------------
 ci/travis_script_integration.sh                 |  3 +-
 integration/integration_test.py                 | 27 +++++++++++-------
 .../src/main/codegen/data/ValueVectorTypes.tdd  | 14 ++++-----
 .../src/main/codegen/includes/vv_imports.ftl    |  1 +
 .../codegen/templates/AbstractFieldReader.java  |  6 ++--
 .../src/main/codegen/templates/ArrowType.java   |  1 +
 .../src/main/codegen/templates/BaseReader.java  |  5 ++--
 .../main/codegen/templates/ComplexReaders.java  |  7 +++++
 .../main/codegen/templates/ComplexWriters.java  |  3 ++
 .../codegen/templates/FixedValueVectors.java    | 18 ++++--------
 .../codegen/templates/HolderReaderImpl.java     | 12 ++++----
 .../src/main/codegen/templates/NullReader.java  |  6 ++--
 .../src/main/codegen/templates/UnionReader.java |  6 ++--
 .../src/main/codegen/templates/UnionVector.java |  1 +
 .../main/codegen/templates/ValueHolders.java    |  3 ++
 .../org/apache/arrow/vector/types/Types.java    | 11 ++++---
 .../apache/arrow/vector/util/DateUtility.java   | 10 +++++++
 .../main/java/org/joda/time/LocalDateTimes.java | 30 ++++++++++++++++++++
 .../complex/writer/TestComplexWriter.java       | 19 ++++++-------
 .../apache/arrow/vector/file/BaseFileTest.java  | 20 ++++++-------
 .../apache/arrow/vector/pojo/TestConvert.java   |  2 +-
 21 files changed, 134 insertions(+), 71 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/ci/travis_script_integration.sh
----------------------------------------------------------------------
diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
index 56f5ab7..6e93ed7 100755
--- a/ci/travis_script_integration.sh
+++ b/ci/travis_script_integration.sh
@@ -18,7 +18,8 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 pushd $ARROW_JAVA_DIR
 
-mvn package
+echo "mvn package"
+mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log &&
false)
 
 popd
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/integration/integration_test.py
----------------------------------------------------------------------
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 661f5c9..6466469 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -544,7 +544,8 @@ class JSONRecordBatch(object):
 
 class JSONFile(object):
 
-    def __init__(self, schema, batches):
+    def __init__(self, name, schema, batches):
+        self.name = name
         self.schema = schema
         self.batches = batches
 
@@ -579,7 +580,7 @@ def get_field(name, type_, nullable=True):
         raise TypeError(dtype)
 
 
-def _generate_file(fields, batch_sizes):
+def _generate_file(name, fields, batch_sizes):
     schema = JSONSchema(fields)
     batches = []
     for size in batch_sizes:
@@ -590,7 +591,7 @@ def _generate_file(fields, batch_sizes):
 
         batches.append(JSONRecordBatch(size, columns))
 
-    return JSONFile(schema, batches)
+    return JSONFile(name, schema, batches)
 
 
 def generate_primitive_case(batch_sizes):
@@ -604,7 +605,7 @@ def generate_primitive_case(batch_sizes):
         fields.append(get_field(type_ + "_nullable", type_, True))
         fields.append(get_field(type_ + "_nonnullable", type_, False))
 
-    return _generate_file(fields, batch_sizes)
+    return _generate_file("primitive", fields, batch_sizes)
 
 
 def generate_datetime_case():
@@ -619,11 +620,11 @@ def generate_datetime_case():
         TimestampType('f7', 'ms'),
         TimestampType('f8', 'us'),
         TimestampType('f9', 'ns'),
-        TimestampType('f10', 'ms', tz='America/New_York')
+        TimestampType('f10', 'ms', tz=None)
     ]
 
     batch_sizes = [7, 10]
-    return _generate_file(fields, batch_sizes)
+    return _generate_file("datetime", fields, batch_sizes)
 
 
 def generate_nested_case():
@@ -637,7 +638,7 @@ def generate_nested_case():
     ]
 
     batch_sizes = [7, 10]
-    return _generate_file(fields, batch_sizes)
+    return _generate_file("nested", fields, batch_sizes)
 
 
 def get_generated_json_files():
@@ -655,7 +656,7 @@ def get_generated_json_files():
 
     generated_paths = []
     for file_obj in file_objs:
-        out_path = os.path.join(temp_dir, guid() + '.json')
+        out_path = os.path.join(temp_dir, 'generated_' + file_obj.name + '.json')
         file_obj.write(out_path)
         generated_paths.append(out_path)
 
@@ -684,11 +685,15 @@ class IntegrationRunner(object):
                                                        consumer.name))
 
         for json_path in self.json_files:
+            print('=====================================================================================')
             print('Testing file {0}'.format(json_path))
+            print('=====================================================================================')
+
+            name = os.path.splitext(os.path.basename(json_path))[0]
 
             # Make the random access file
             print('-- Creating binary inputs')
-            producer_file_path = os.path.join(self.temp_dir, guid())
+            producer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.json_to_arrow')
             producer.json_to_file(json_path, producer_file_path)
 
             # Validate the file
@@ -696,8 +701,8 @@ class IntegrationRunner(object):
             consumer.validate(json_path, producer_file_path)
 
             print('-- Validating stream')
-            producer_stream_path = os.path.join(self.temp_dir, guid())
-            consumer_file_path = os.path.join(self.temp_dir, guid())
+            producer_stream_path = os.path.join(self.temp_dir, guid() + '_' + name + '.arrow_to_stream')
+            consumer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.stream_to_arrow')
             producer.file_to_stream(producer_file_path,
                                     producer_stream_path)
             consumer.stream_to_file(producer_stream_path,

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index d472b55..ca6d9ec 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -59,7 +59,7 @@
         { class: "DateDay" },
         { class: "IntervalYear", javaType: "int", friendlyType: "Period" },
         { class: "TimeSec" },
-        { class: "TimeMilli", javaType: "int", friendlyType: "DateTime" }
+        { class: "TimeMilli", javaType: "int", friendlyType: "LocalDateTime" }
       ]
     },
     {
@@ -71,12 +71,12 @@
       minor: [
         { class: "BigInt"},
         { class: "UInt8" },
-        { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value",
type: "double"}], },
-        { class: "DateMilli", javaType: "long", friendlyType: "DateTime" },
-        { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime"
},
-        { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime"
},
-        { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime"
},
-        { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime"
},
+        { class: "Float8",         javaType: "double", boxedType: "Double",             
                fields: [{name: "value", type: "double"}], },
+        { class: "DateMilli",      javaType: "long",                      friendlyType: "LocalDateTime"
},
+        { class: "TimeStampSec",   javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime"
},
+        { class: "TimeStampMilli", javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime"
},
+        { class: "TimeStampMicro", javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime"
},
+        { class: "TimeStampNano",  javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime"
},
         { class: "TimeMicro" },
         { class: "TimeNano" }
       ]

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl
index 9b4b79b..e723e7d 100644
--- a/java/vector/src/main/codegen/includes/vv_imports.ftl
+++ b/java/vector/src/main/codegen/includes/vv_imports.ftl
@@ -57,6 +57,7 @@ import java.math.BigDecimal;
 import java.math.BigInteger;
 
 import org.joda.time.DateTime;
+import org.joda.time.LocalDateTime;
 import org.joda.time.Period;
 
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/AbstractFieldReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldReader.java b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
index 79d4c12..b16ee16 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldReader.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
@@ -26,8 +26,8 @@ package org.apache.arrow.vector.complex.impl;
 
 <#include "/@includes/vv_imports.ftl" />
 
-/*
- * This class is generated using freemarker and the ${.template_name} template.
+/**
+ * Source code generated using FreeMarker template ${.template_name}
  */
 @SuppressWarnings("unused")
 abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
@@ -51,7 +51,7 @@ abstract class AbstractFieldReader extends AbstractBaseReader implements
FieldRe
   }
 
   <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
-          "Character", "DateTime", "Period", "Double", "Float",
+          "Character", "LocalDateTime", "Period", "Double", "Float",
           "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
   <#assign safeType=friendlyType />
   <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/ArrowType.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java
index dc99aad..9374630 100644
--- a/java/vector/src/main/codegen/templates/ArrowType.java
+++ b/java/vector/src/main/codegen/templates/ArrowType.java
@@ -38,6 +38,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
 
 /**
  * Arrow types
+ * Source code generated using FreeMarker template ${.template_name}
  **/
 @JsonTypeInfo(
   use = JsonTypeInfo.Id.NAME,

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/BaseReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/BaseReader.java b/java/vector/src/main/codegen/templates/BaseReader.java
index 72fea58..ea3286e 100644
--- a/java/vector/src/main/codegen/templates/BaseReader.java
+++ b/java/vector/src/main/codegen/templates/BaseReader.java
@@ -26,8 +26,9 @@ package org.apache.arrow.vector.complex.reader;
 
 <#include "/@includes/vv_imports.ftl" />
 
-
-
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public interface BaseReader extends Positionable{
   Field getField();

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/ComplexReaders.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java
index d537445..38cd1bf 100644
--- a/java/vector/src/main/codegen/templates/ComplexReaders.java
+++ b/java/vector/src/main/codegen/templates/ComplexReaders.java
@@ -47,6 +47,9 @@ package org.apache.arrow.vector.complex.impl;
 
 <#include "/@includes/vv_imports.ftl" />
 
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public class ${name}ReaderImpl extends AbstractFieldReader {
   
@@ -123,12 +126,16 @@ public class ${name}ReaderImpl extends AbstractFieldReader {
 package org.apache.arrow.vector.complex.reader;
 
 <#include "/@includes/vv_imports.ftl" />
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public interface ${name}Reader extends BaseReader{
   
   public void read(${minor.class?cap_first}Holder h);
   public void read(Nullable${minor.class?cap_first}Holder h);
   public Object readObject();
+  // read friendly type
   public ${friendlyType} read${safeType}();
   public boolean isSet();
   public void copyAsValue(${minor.class}Writer writer);

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/ComplexWriters.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index 3457545..c23b89d 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -139,6 +139,9 @@ public class ${eName}WriterImpl extends AbstractFieldWriter {
 package org.apache.arrow.vector.complex.writer;
 
 <#include "/@includes/vv_imports.ftl" />
+/*
+ * This class is generated using FreeMarker on the ${.template_name} template.
+ */
 @SuppressWarnings("unused")
 public interface ${eName}Writer extends BaseWriter {
   public void write(${minor.class}Holder h);

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 05faaae..f403ecf 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -495,16 +495,14 @@ public final class ${minor.class}Vector extends BaseDataValueVector
implements F
     <#elseif minor.class == "DateMilli">
     @Override
     public ${friendlyType} getObject(int index) {
-      org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
-      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
       return date;
     }
 
     <#elseif minor.class == "TimeMilli">
     @Override
     public ${friendlyType} getObject(int index) {
-      org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
-      time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
       return time;
     }
 
@@ -512,16 +510,14 @@ public final class ${minor.class}Vector extends BaseDataValueVector
implements F
     @Override
     public ${friendlyType} getObject(int index) {
       long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
-      org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC);
-      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC);
       return date;
     }
 
     <#elseif minor.class == "TimeStampMilli">
     @Override
     public ${friendlyType} getObject(int index) {
-        org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
-        date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+        org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
         return date;
     }
 
@@ -530,8 +526,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements
F
     public ${friendlyType} getObject(int index) {
       // value is truncated when converting microseconds to milliseconds in order to use
DateTime type
       long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index));
-      org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC);
-      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC);
       return date;
     }
 
@@ -540,8 +535,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements
F
     public ${friendlyType} getObject(int index) {
       // value is truncated when converting nanoseconds to milliseconds in order to use DateTime
type
       long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index));
-      org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC);
-      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
       return date;
     }
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/HolderReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
index d66577b..e990fcc 100644
--- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java
+++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -84,7 +84,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader
{
     
   }
 
-@Override
+  @Override
   public void read(${name}Holder h) {
   <#list fields as field>
     h.${field.name} = holder.${field.name};
@@ -99,7 +99,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader
{
     h.isSet = isSet() ? 1 : 0;
   }
 
-
+  // read friendly type
   @Override
   public ${friendlyType} read${safeType}(){
 <#if nullMode == "Nullable">
@@ -114,15 +114,15 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader
{
       byte[] value = new byte [length];
       holder.buffer.getBytes(holder.start, value, 0, length);
 
-<#if minor.class == "VarBinary">
+  <#if minor.class == "VarBinary">
       return value;
-<#elseif minor.class == "Var16Char">
+  <#elseif minor.class == "Var16Char">
       return new String(value);
-<#elseif minor.class == "VarChar">
+  <#elseif minor.class == "VarChar">
       Text text = new Text();
       text.set(value);
       return text;
-</#if>
+  </#if>
 
 <#elseif minor.class == "Interval">
       Period p = new Period();

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/NullReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/NullReader.java b/java/vector/src/main/codegen/templates/NullReader.java
index ba0c088..7c75b3a 100644
--- a/java/vector/src/main/codegen/templates/NullReader.java
+++ b/java/vector/src/main/codegen/templates/NullReader.java
@@ -29,7 +29,9 @@ package org.apache.arrow.vector.complex.impl;
 
 <#include "/@includes/vv_imports.ftl" />
 
-
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public class NullReader extends AbstractBaseReader implements FieldReader{
   
@@ -127,7 +129,7 @@ public class NullReader extends AbstractBaseReader implements FieldReader{
   }
   
   <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", 
-          "Character", "DateTime", "Period", "Double", "Float",
+          "Character", "LocalDateTime", "Period", "Double", "Float",
           "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
   <#assign safeType=friendlyType />
   <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/UnionReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java
index c56e95c..0b5a209 100644
--- a/java/vector/src/main/codegen/templates/UnionReader.java
+++ b/java/vector/src/main/codegen/templates/UnionReader.java
@@ -28,7 +28,9 @@ import org.apache.arrow.vector.types.Types.MinorType;
 package org.apache.arrow.vector.complex.impl;
 
 <#include "/@includes/vv_imports.ftl" />
-
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public class UnionReader extends AbstractFieldReader {
 
@@ -122,7 +124,7 @@ public class UnionReader extends AbstractFieldReader {
   }
 
   <#list ["Object", "Integer", "Long", "Boolean",
-          "Character", "DateTime", "Double", "Float",
+          "Character", "LocalDateTime", "Double", "Float",
           "Text", "Byte", "Short", "byte[]"] as friendlyType>
   <#assign safeType=friendlyType />
   <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index d70cbae..9d5dee5 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -50,6 +50,7 @@ import static org.apache.arrow.vector.types.UnionMode.Sparse;
  *
  * For performance reasons, UnionVector stores a cached reference to each subtype vector,
to avoid having to do the map lookup
  * each time the vector is accessed.
+ * Source code generated using FreeMarker template ${.template_name}
  */
 public class UnionVector implements FieldVector {
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/codegen/templates/ValueHolders.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ValueHolders.java b/java/vector/src/main/codegen/templates/ValueHolders.java
index d744c52..a474b69 100644
--- a/java/vector/src/main/codegen/templates/ValueHolders.java
+++ b/java/vector/src/main/codegen/templates/ValueHolders.java
@@ -29,6 +29,9 @@ package org.apache.arrow.vector.holders;
 
 <#include "/@includes/vv_imports.ftl" />
 
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 public final class ${className} implements ValueHolder{
   
     <#if mode.name == "Repeated">

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 6023f1c..d5076d8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -241,7 +241,7 @@ public class Types {
       }
     },
     // time in second from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
-    TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, "UTC")) {
+    TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, null)) {
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator,
CallBack schemaChangeCallback) {
         return new NullableTimeStampSecVector(name, fieldType, allocator);
@@ -253,7 +253,7 @@ public class Types {
       }
     },
     // time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
-    TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, "UTC"))
{
+    TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, null))
{
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator,
CallBack schemaChangeCallback) {
         return new NullableTimeStampMilliVector(name, fieldType, allocator);
@@ -265,7 +265,7 @@ public class Types {
       }
     },
     // time in microsecond from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
-    TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, "UTC"))
{
+    TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, null))
{
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator,
CallBack schemaChangeCallback) {
         return new NullableTimeStampMicroVector(name, fieldType, allocator);
@@ -277,7 +277,7 @@ public class Types {
       }
     },
     // time in nanosecond from the Unix epoch, 00:00:00.000000000 on 1 January 1970, UTC.
-    TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, "UTC"))
{
+    TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, null))
{
       @Override
       public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator,
CallBack schemaChangeCallback) {
         return new NullableTimeStampNanoVector(name, fieldType, allocator);
@@ -580,6 +580,9 @@ public class Types {
       }
 
       @Override public MinorType visit(Timestamp type) {
+        if (type.getTimezone() != null) {
+          throw new IllegalArgumentException("only timezone-less timestamps are supported
for now: " + type);
+        }
         switch (type.getUnit()) {
           case SECOND:
             return MinorType.TIMESTAMPSEC;

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
index 1f8ce06..8aad417 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java
@@ -18,6 +18,9 @@
 
 package org.apache.arrow.vector.util;
 
+import org.joda.time.DateTimeZone;
+import org.joda.time.LocalDateTime;
+import org.joda.time.LocalDateTimes;
 import org.joda.time.Period;
 import org.joda.time.format.DateTimeFormat;
 import org.joda.time.format.DateTimeFormatter;
@@ -679,4 +682,11 @@ public class DateUtility {
       (period.getMillis());
     }
 
+    public static long toMillis(LocalDateTime localDateTime) {
+      return LocalDateTimes.getLocalMillis(localDateTime);
+    }
+
+    public static int toMillisOfDay(final LocalDateTime localDateTime) {
+      return localDateTime.toDateTime(DateTimeZone.UTC).millisOfDay().get();
+    }
 }

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/main/java/org/joda/time/LocalDateTimes.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/joda/time/LocalDateTimes.java b/java/vector/src/main/java/org/joda/time/LocalDateTimes.java
new file mode 100644
index 0000000..e4f999e
--- /dev/null
+++ b/java/vector/src/main/java/org/joda/time/LocalDateTimes.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.joda.time;
+
+/**
+ * Workaround to access package protected fields in JODA
+ *
+ */
+public class LocalDateTimes {
+
+  public static long getLocalMillis(LocalDateTime localDateTime) {
+    return localDateTime.getLocalMillis();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 99ba19b..aba65db 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -56,8 +56,7 @@ import org.apache.arrow.vector.util.JsonStringArrayList;
 import org.apache.arrow.vector.util.JsonStringHashMap;
 import org.apache.arrow.vector.util.Text;
 import org.apache.arrow.vector.util.TransferPair;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
+import org.joda.time.LocalDateTime;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -602,10 +601,10 @@ public class TestComplexWriter {
     final long expectedMicros = 981173106123456L;
     final long expectedMillis = 981173106123L;
     final long expectedSecs = 981173106L;
-    final DateTime expectedSecDateTime = new DateTime(2001, 2, 3, 4, 5, 6, 0).withZoneRetainFields(DateTimeZone.getDefault());
-    final DateTime expectedMilliDateTime = new DateTime(2001, 2, 3, 4, 5, 6, 123).withZoneRetainFields(DateTimeZone.getDefault());
-    final DateTime expectedMicroDateTime = expectedMilliDateTime;
-    final DateTime expectedNanoDateTime = expectedMilliDateTime;
+    final LocalDateTime expectedSecDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 0);
+    final LocalDateTime expectedMilliDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123);
+    final LocalDateTime expectedMicroDateTime = expectedMilliDateTime;
+    final LocalDateTime expectedNanoDateTime = expectedMilliDateTime;
 
     // write
     MapVector parent = new MapVector("parent", allocator, null);
@@ -650,28 +649,28 @@ public class TestComplexWriter {
 
     FieldReader secReader = rootReader.reader("sec");
     secReader.setPosition(0);
-    DateTime secDateTime = secReader.readDateTime();
+    LocalDateTime secDateTime = secReader.readLocalDateTime();
     Assert.assertEquals(expectedSecDateTime, secDateTime);
     long secLong = secReader.readLong();
     Assert.assertEquals(expectedSecs, secLong);
 
     FieldReader milliReader = rootReader.reader("milli");
     milliReader.setPosition(1);
-    DateTime milliDateTime = milliReader.readDateTime();
+    LocalDateTime milliDateTime = milliReader.readLocalDateTime();
     Assert.assertEquals(expectedMilliDateTime, milliDateTime);
     long milliLong = milliReader.readLong();
     Assert.assertEquals(expectedMillis, milliLong);
 
     FieldReader microReader = rootReader.reader("micro");
     microReader.setPosition(2);
-    DateTime microDateTime = microReader.readDateTime();
+    LocalDateTime microDateTime = microReader.readLocalDateTime();
     Assert.assertEquals(expectedMicroDateTime, microDateTime);
     long microLong = microReader.readLong();
     Assert.assertEquals(expectedMicros, microLong);
 
     FieldReader nanoReader = rootReader.reader("nano");
     nanoReader.setPosition(3);
-    DateTime nanoDateTime = nanoReader.readDateTime();
+    LocalDateTime nanoDateTime = nanoReader.readLocalDateTime();
     Assert.assertEquals(expectedNanoDateTime, nanoDateTime);
     long nanoLong = nanoReader.readLong();
     Assert.assertEquals(expectedNanos, nanoLong);

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
index 5ca083a..5cc36a3 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -39,8 +39,8 @@ import org.apache.arrow.vector.complex.writer.IntWriter;
 import org.apache.arrow.vector.complex.writer.TimeMilliWriter;
 import org.apache.arrow.vector.complex.writer.TimeStampMilliWriter;
 import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder;
-import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.joda.time.LocalDateTime;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -144,8 +144,8 @@ public class BaseFileTest {
     }
   }
 
-  private DateTime makeDateTimeFromCount(int i) {
-    return new DateTime(2000 + i, 1 + i, 1 + i, i, i, i, i, DateTimeZone.UTC);
+  private LocalDateTime makeDateTimeFromCount(int i) {
+    return new LocalDateTime(2000 + i, 1 + i, 1 + i, i, i, i, i);
   }
 
   protected void writeDateTimeData(int count, NullableMapVector parent) {
@@ -156,17 +156,17 @@ public class BaseFileTest {
     TimeMilliWriter timeWriter = rootWriter.timeMilli("time");
     TimeStampMilliWriter timeStampMilliWriter = rootWriter.timeStampMilli("timestamp-milli");
     for (int i = 0; i < count; i++) {
-      DateTime dt = makeDateTimeFromCount(i);
+      LocalDateTime dt = makeDateTimeFromCount(i);
       // Number of days in milliseconds since epoch, stored as 64-bit integer, only date
part is used
       dateWriter.setPosition(i);
-      long dateLong = dt.minusMillis(dt.getMillisOfDay()).getMillis();
+      long dateLong = org.apache.arrow.vector.util.DateUtility.toMillis(dt.minusMillis(dt.getMillisOfDay()));
       dateWriter.writeDateMilli(dateLong);
       // Time is a value in milliseconds since midnight, stored as 32-bit integer
       timeWriter.setPosition(i);
       timeWriter.writeTimeMilli(dt.getMillisOfDay());
       // Timestamp is milliseconds since the epoch, stored as 64-bit integer
       timeStampMilliWriter.setPosition(i);
-      timeStampMilliWriter.writeTimeStampMilli(dt.getMillis());
+      timeStampMilliWriter.writeTimeStampMilli(org.apache.arrow.vector.util.DateUtility.toMillis(dt));
     }
     writer.setValueCount(count);
   }
@@ -176,13 +176,13 @@ public class BaseFileTest {
     printVectors(root.getFieldVectors());
     for (int i = 0; i < count; i++) {
       long dateVal = ((NullableDateMilliVector)root.getVector("date")).getAccessor().get(i);
-      DateTime dt = makeDateTimeFromCount(i);
-      DateTime dateExpected = dt.minusMillis(dt.getMillisOfDay());
-      Assert.assertEquals(dateExpected.getMillis(), dateVal);
+      LocalDateTime dt = makeDateTimeFromCount(i);
+      LocalDateTime dateExpected = dt.minusMillis(dt.getMillisOfDay());
+      Assert.assertEquals(org.apache.arrow.vector.util.DateUtility.toMillis(dateExpected),
dateVal);
       long timeVal = ((NullableTimeMilliVector)root.getVector("time")).getAccessor().get(i);
       Assert.assertEquals(dt.getMillisOfDay(), timeVal);
       Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i);
-      Assert.assertTrue(dt.withZoneRetainFields(DateTimeZone.getDefault()).equals(timestampMilliVal));
+      Assert.assertEquals(dt, timestampMilliVal);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/316c63db/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index 824c62a..f9c8f72 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -81,7 +81,7 @@ public class TestConvert {
         new Field("child4.1", true, Utf8.INSTANCE, null)
         )));
     childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] {
MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of(
-        new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null),
+        new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND, null), null),
         new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
         )));
     Schema initialSchema = new Schema(childrenBuilder.build());


Mime
View raw message