hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1656519 [1/2] - in /hive/trunk: ./ common/ common/src/java/org/apache/hive/common/util/ common/src/test/org/apache/hive/common/util/ data/files/ hbase-handler/src/java/org/apache/hadoop/hive/hbase/ hbase-handler/src/java/org/apache/hadoop/...
Date Mon, 02 Feb 2015 17:04:42 GMT
Author: hashutosh
Date: Mon Feb  2 17:04:41 2015
New Revision: 1656519

URL: http://svn.apache.org/r1656519
Log:
HIVE-9298 : Support reading alternate timestamp formats (Jason Dere via Ashutosh Chauhan)

Added:
    hive/trunk/common/src/java/org/apache/hive/common/util/TimestampParser.java
    hive/trunk/common/src/test/org/apache/hive/common/util/TestHiveStringUtils.java
    hive/trunk/common/src/test/org/apache/hive/common/util/TestTimestampParser.java
    hive/trunk/data/files/ts_formats.txt
    hive/trunk/hbase-handler/src/test/queries/positive/hbase_timestamp_format.q
    hive/trunk/hbase-handler/src/test/results/positive/hbase_timestamp_format.q.out
    hive/trunk/ql/src/test/queries/clientpositive/timestamp_formats.q
    hive/trunk/ql/src/test/results/clientpositive/timestamp_formats.q.out
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParameters.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParametersImpl.java
Modified:
    hive/trunk/common/pom.xml
    hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
    hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java
    hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseLazyObjectFactory.java
    hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/AvroHBaseValueFactory.java
    hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java
    hive/trunk/pom.xml
    hive/trunk/ql/pom.xml
    hive/trunk/serde/if/serde.thrift
    hive/trunk/serde/src/gen/thrift/gen-cpp/serde_constants.cpp
    hive/trunk/serde/src/gen/thrift/gen-cpp/serde_constants.h
    hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java
    hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/ThriftTestObj.java
    hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java
    hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MegaStruct.java
    hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/PropValueUnion.java
    hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java
    hive/trunk/serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php
    hive/trunk/serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py
    hive/trunk/serde/src/gen/thrift/gen-rb/serde_constants.rb
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyUnionObjectInspector.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyTimestampObjectInspector.java

Modified: hive/trunk/common/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/common/pom.xml?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/common/pom.xml (original)
+++ hive/trunk/common/pom.xml Mon Feb  2 17:04:41 2015
@@ -56,6 +56,11 @@
       <version>${commons-logging.version}</version>
     </dependency>
     <dependency>
+      <groupId>joda-time</groupId>
+      <artifactId>joda-time</artifactId>
+      <version>${joda.version}</version>
+    </dependency>
+    <dependency>
       <groupId>log4j</groupId>
       <artifactId>log4j</artifactId>
       <version>${log4j.version}</version>

Modified: hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java (original)
+++ hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java Mon Feb  2 17:04:41 2015
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.classification.InterfaceAudience;
 import org.apache.hadoop.hive.common.classification.InterfaceStability;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.StringUtils;
 
 /**
  * HiveStringUtils
@@ -492,6 +493,34 @@ public class HiveStringUtils {
   }
 
   /**
+   * Split a string using the default separator/escape character,
+   * then unescape the resulting array of strings
+   * @param str
+   * @return an array of unescaped strings
+   */
+  public static String[] splitAndUnEscape(String str) {
+    return splitAndUnEscape(str, ESCAPE_CHAR, COMMA);
+  }
+
+  /**
+   * Split a string using the specified separator/escape character,
+   * then unescape the resulting array of strings using the same escape/separator.
+   * @param str a string that may have escaped separator
+   * @param escapeChar a char that be used to escape the separator
+   * @param separator a separator char
+   * @return an array of unescaped strings
+   */
+  public static String[] splitAndUnEscape(String str, char escapeChar, char separator) {
+    String[] result = split(str, escapeChar, separator);
+    if (result != null) {
+      for (int idx = 0; idx < result.length; ++idx) {
+        result[idx] = unEscapeString(result[idx], escapeChar, separator);
+      }
+    }
+    return result;
+  }
+
+  /**
    * Finds the first occurrence of the separator character ignoring the escaped
    * separators starting from the index. Note the substring between the index
    * and the position of the separator is passed.

Added: hive/trunk/common/src/java/org/apache/hive/common/util/TimestampParser.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hive/common/util/TimestampParser.java?rev=1656519&view=auto
==============================================================================
--- hive/trunk/common/src/java/org/apache/hive/common/util/TimestampParser.java (added)
+++ hive/trunk/common/src/java/org/apache/hive/common/util/TimestampParser.java Mon Feb  2 17:04:41 2015
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.common.util;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.joda.time.DateTime;
+import org.joda.time.MutableDateTime;
+import org.joda.time.DateTimeFieldType;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.joda.time.format.DateTimeFormatterBuilder;
+import org.joda.time.format.DateTimeParser;
+import org.joda.time.format.DateTimeParserBucket;
+
+/**
+ * Timestamp parser using Joda DateTimeFormatter. Parser accepts 0 or more date time format
+ * patterns. If no format patterns are provided it will default to the normal Timestamp parsing.
+ * Datetime formats are compatible with Java SimpleDateFormat. Also added special case pattern
+ * "millis" to parse the string as milliseconds since Unix epoch.
+ * Since this uses Joda DateTimeFormatter, this parser should be thread safe.
+ */
+public class TimestampParser {
+
+  protected final static String[] stringArray = new String[] {};
+  protected final static String millisFormatString = "millis";
+  protected final static DateTime startingDateValue = new DateTime(1970, 1, 1, 0, 0, 0, 0);
+
+  protected String[] formatStrings = null;
+  protected DateTimeFormatter fmt = null;
+
+  public TimestampParser() {
+  }
+
+  public TimestampParser(TimestampParser tsParser) {
+    this(tsParser.formatStrings == null ?
+        null : Arrays.copyOf(tsParser.formatStrings, tsParser.formatStrings.length));
+  }
+
+  public TimestampParser(List<String> formatStrings) {
+    this(formatStrings == null ? null : formatStrings.toArray(stringArray));
+  }
+
+  public TimestampParser(String[] formatStrings) {
+    this.formatStrings = formatStrings;
+
+    // create formatter that includes all of the input patterns
+    if (formatStrings != null && formatStrings.length > 0) {
+      DateTimeParser[] parsers = new DateTimeParser[formatStrings.length];
+      for (int idx = 0; idx < formatStrings.length; ++idx) {
+        String formatString = formatStrings[idx];
+        if (formatString.equalsIgnoreCase(millisFormatString)) {
+          // Use milliseconds parser if pattern matches our special-case millis pattern string
+          parsers[idx] = new MillisDateFormatParser();
+        } else {
+          parsers[idx] = DateTimeFormat.forPattern(formatString).getParser();
+        }
+      }
+      fmt = new DateTimeFormatterBuilder().append(null, parsers).toFormatter();
+    }
+  }
+
+  /**
+   * Parse the input string and return a timestamp value
+   * @param strValue
+   * @return
+   * @throws IllegalArgumentException if input string cannot be parsed into timestamp
+   */
+  public Timestamp parseTimestamp(String strValue) throws IllegalArgumentException {
+    if (fmt != null) {
+      // reset value in case any date fields are missing from the date pattern
+      MutableDateTime mdt = new MutableDateTime(startingDateValue);
+
+      // Using parseInto() avoids throwing exception when parsing,
+      // allowing fallback to default timestamp parsing if custom patterns fail.
+      int ret = fmt.parseInto(mdt, strValue, 0);
+      // Only accept parse results if we parsed the entire string
+      if (ret == strValue.length()) {
+        return new Timestamp(mdt.getMillis());
+      }
+    }
+
+    // Otherwise try default timestamp parsing
+    return Timestamp.valueOf(strValue);
+  }
+
+  /**
+   * DateTimeParser to parse the date string as the millis since Unix epoch
+   */
+  public static class MillisDateFormatParser implements DateTimeParser {
+    private static final ThreadLocal<Matcher> numericMatcher = new ThreadLocal<Matcher>() {
+      @Override
+      protected Matcher initialValue() {
+        return Pattern.compile("(-?\\d+)(\\.\\d+)?$").matcher("");
+      }
+    };
+
+    private final static DateTimeFieldType[] dateTimeFields = {
+      DateTimeFieldType.year(),
+      DateTimeFieldType.monthOfYear(),
+      DateTimeFieldType.dayOfMonth(),
+      DateTimeFieldType.hourOfDay(),
+      DateTimeFieldType.minuteOfHour(),
+      DateTimeFieldType.secondOfMinute(),
+      DateTimeFieldType.millisOfSecond()
+    };
+
+    public int estimateParsedLength() {
+      return 13; // Shouldn't hit 14 digits until year 2286
+    }
+
+    public int parseInto(DateTimeParserBucket bucket, String text, int position) {
+      String substr = text.substring(position);
+      Matcher matcher = numericMatcher.get();
+      matcher.reset(substr);
+      if (!matcher.matches()) {
+        return -1;
+      }
+
+      // Joda DateTime only has precision to millis, cut off any fractional portion
+      long millis = Long.parseLong(matcher.group(1));
+      DateTime dt = new DateTime(millis);
+      for (DateTimeFieldType field : dateTimeFields) {
+        bucket.saveField(field, dt.get(field));
+      }
+      return substr.length();
+    }
+  }
+}

Added: hive/trunk/common/src/test/org/apache/hive/common/util/TestHiveStringUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/test/org/apache/hive/common/util/TestHiveStringUtils.java?rev=1656519&view=auto
==============================================================================
--- hive/trunk/common/src/test/org/apache/hive/common/util/TestHiveStringUtils.java (added)
+++ hive/trunk/common/src/test/org/apache/hive/common/util/TestHiveStringUtils.java Mon Feb  2 17:04:41 2015
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.common.util;
+
+import static org.junit.Assert.*;
+
+import java.util.Arrays;
+
+import org.junit.Test;
+
+public class TestHiveStringUtils {
+  @Test
+  public void testSplitAndUnEscape() throws Exception {
+    splitAndUnEscapeTestCase(
+        null, null);
+
+    splitAndUnEscapeTestCase(
+        "'single element'",
+        new String[] {
+            "'single element'"
+        });
+
+    splitAndUnEscapeTestCase(
+        "yyyy-MM-dd'T'HH:mm:ss,yyyy-MM-dd'T'HH:mm:ss.S",
+        new String[] {
+            "yyyy-MM-dd'T'HH:mm:ss",
+            "yyyy-MM-dd'T'HH:mm:ss.S"
+        });
+
+    splitAndUnEscapeTestCase(
+        "single\\,element",
+        new String[] {
+            "single,element"
+        });
+    splitAndUnEscapeTestCase(
+        "element\\,one\\\\,element\\\\two\\\\\\,",
+        new String[] {
+            "element,one\\",
+            "element\\two\\,"
+        });
+  }
+
+  public void splitAndUnEscapeTestCase(String testValue, String[] expectedResults) throws Exception {
+    String[] testResults = HiveStringUtils.splitAndUnEscape(testValue);
+    assertTrue(Arrays.toString(expectedResults) + " == " + Arrays.toString(testResults),
+        Arrays.equals(expectedResults, testResults));
+  }
+}

Added: hive/trunk/common/src/test/org/apache/hive/common/util/TestTimestampParser.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/test/org/apache/hive/common/util/TestTimestampParser.java?rev=1656519&view=auto
==============================================================================
--- hive/trunk/common/src/test/org/apache/hive/common/util/TestTimestampParser.java (added)
+++ hive/trunk/common/src/test/org/apache/hive/common/util/TestTimestampParser.java Mon Feb  2 17:04:41 2015
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.common.util;
+
+import java.sql.Timestamp;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.*;
+import org.junit.Test;
+
+public class TestTimestampParser {
+  public static class ValidTimestampCase {
+    String strValue;
+    Timestamp expectedValue;
+
+    public ValidTimestampCase(String strValue, Timestamp expectedValue) {
+      this.strValue = strValue;
+      this.expectedValue = expectedValue;
+    }
+  }
+
+  static void testValidCases(TimestampParser tp, ValidTimestampCase[] validCases) {
+    for (ValidTimestampCase validCase : validCases) {
+      Timestamp ts = tp.parseTimestamp(validCase.strValue);
+      assertEquals("Parsing " + validCase.strValue, validCase.expectedValue, ts);
+    }
+  }
+
+  static void testInvalidCases(TimestampParser tp, String[] invalidCases) {
+    for (String invalidString : invalidCases) {
+      try {
+        Timestamp ts = tp.parseTimestamp(invalidString);
+        fail("Expected exception parsing " + invalidString + ", but parsed value to " + ts);
+      } catch (IllegalArgumentException err) {
+        // Exception expected
+      }
+    }
+  }
+
+  @Test
+  public void testDefault() {
+    // No timestamp patterns, should default to normal timestamp format
+    TimestampParser tp = new TimestampParser();
+    ValidTimestampCase[] validCases = {
+        new ValidTimestampCase("1945-12-31 23:59:59.0",
+            Timestamp.valueOf("1945-12-31 23:59:59.0")),
+        new ValidTimestampCase("1945-12-31 23:59:59.1234",
+            Timestamp.valueOf("1945-12-31 23:59:59.1234")),
+        new ValidTimestampCase("1970-01-01 00:00:00",
+            Timestamp.valueOf("1970-01-01 00:00:00")),
+    };
+
+    String[] invalidCases = {
+        "1945-12-31T23:59:59",
+        "12345",
+    };
+
+    testValidCases(tp, validCases);
+    testInvalidCases(tp, invalidCases);
+  }
+
+  @Test
+  public void testPattern1() {
+    // Joda pattern matching expects fractional seconds length to match
+    // the number of 'S' in the pattern. So if you want to match .1, .12, .123,
+    // you need 3 different patterns with .S, .SS, .SSS
+    String[] patterns = {
+        // ISO-8601 timestamps
+        "yyyy-MM-dd'T'HH:mm:ss",
+        "yyyy-MM-dd'T'HH:mm:ss.S",
+        "yyyy-MM-dd'T'HH:mm:ss.SS",
+        "yyyy-MM-dd'T'HH:mm:ss.SSS",
+        "yyyy-MM-dd'T'HH:mm:ss.SSSS",
+    };
+    TimestampParser tp = new TimestampParser(patterns);
+
+    ValidTimestampCase[] validCases = {
+        new ValidTimestampCase("1945-12-31T23:59:59.0",
+            Timestamp.valueOf("1945-12-31 23:59:59.0")),
+        new ValidTimestampCase("2001-01-01 00:00:00.100",
+            Timestamp.valueOf("2001-01-01 00:00:00.100")),
+        new ValidTimestampCase("2001-01-01 00:00:00.001",
+            Timestamp.valueOf("2001-01-01 00:00:00.001")),
+        // Joda parsing only supports up to millisecond precision
+        new ValidTimestampCase("1945-12-31T23:59:59.1234",
+            Timestamp.valueOf("1945-12-31 23:59:59.123")),
+        new ValidTimestampCase("1970-01-01T00:00:00",
+            Timestamp.valueOf("1970-01-01 00:00:00")),
+        new ValidTimestampCase("1970-4-5T6:7:8",
+             Timestamp.valueOf("1970-04-05 06:07:08")),
+
+        // Default timestamp format still works?
+        new ValidTimestampCase("2001-01-01 00:00:00",
+            Timestamp.valueOf("2001-01-01 00:00:00")),
+        new ValidTimestampCase("1945-12-31 23:59:59.1234",
+            Timestamp.valueOf("1945-12-31 23:59:59.1234")),
+    };
+
+    String[] invalidCases = {
+        "1945-12-31-23:59:59",
+        "1945-12-31T23:59:59.12345", // our pattern didn't specify 5 decimal places
+        "12345",
+    };
+
+    testValidCases(tp, validCases);
+    testInvalidCases(tp, invalidCases);
+  }
+
+  @Test
+  public void testMillisParser() {
+    String[] patterns = {
+        "millis",
+        // Also try other patterns
+        "yyyy-MM-dd'T'HH:mm:ss",
+    };
+    TimestampParser tp = new TimestampParser(patterns);
+
+    ValidTimestampCase[] validCases = {
+        new ValidTimestampCase("0", new Timestamp(0)),
+        new ValidTimestampCase("-1000000", new Timestamp(-1000000)),
+        new ValidTimestampCase("1420509274123", new Timestamp(1420509274123L)),
+        new ValidTimestampCase("1420509274123.456789", new Timestamp(1420509274123L)),
+
+        // Other format pattern should also work
+        new ValidTimestampCase("1945-12-31T23:59:59",
+            Timestamp.valueOf("1945-12-31 23:59:59")),
+    };
+
+    String[] invalidCases = {
+        "1945-12-31-23:59:59",
+        "1945-12-31T23:59:59.12345", // our pattern didn't specify 5 decimal places
+        "1420509274123-",
+    };
+
+    testValidCases(tp, validCases);
+    testInvalidCases(tp, invalidCases);
+  }
+
+  @Test
+  public void testPattern2() {
+    // Pattern does not contain all date fields
+    String[] patterns = {
+        "HH:mm",
+        "MM:dd:ss",
+    };
+    TimestampParser tp = new TimestampParser(patterns);
+
+    ValidTimestampCase[] validCases = {
+        new ValidTimestampCase("05:06",
+            Timestamp.valueOf("1970-01-01 05:06:00")),
+        new ValidTimestampCase("05:06:07",
+            Timestamp.valueOf("1970-05-06 00:00:07")),
+    };
+
+    String[] invalidCases = {
+        "1945-12-31T23:59:59",
+        "1945:12:31-",
+        "12345",
+    };
+
+    testValidCases(tp, validCases);
+    testInvalidCases(tp, invalidCases);
+  }
+}

Added: hive/trunk/data/files/ts_formats.txt
URL: http://svn.apache.org/viewvc/hive/trunk/data/files/ts_formats.txt?rev=1656519&view=auto
==============================================================================
--- hive/trunk/data/files/ts_formats.txt (added)
+++ hive/trunk/data/files/ts_formats.txt Mon Feb  2 17:04:41 2015
@@ -0,0 +1,21 @@
+2011-01-01 01:01:01.1111111112011-01-01 01:01:01.1111111112011-01-01T01:01:01.1111111112011-01-01T01:01:01.1111111112011-01-01T01:01:012011-01-01T01:01:01
+2012-02-02 02:02:02.2222222222012-02-02 02:02:02.2222222222012-02-02T02:02:02.2222222222012-02-02T02:02:02.2222222222012-02-02T02:02:022012-02-02T02:02:02
+2013-03-03 03:03:03.3333333332013-03-03 03:03:03.3333333332013-03-03T03:03:03.3333333332013-03-03T03:03:03.3333333332013-03-03T03:03:032013-03-03T03:03:03
+2014-04-04 04:04:04.4444444442014-04-04 04:04:04.4444444442014-04-04T04:04:04.4444444442014-04-04T04:04:04.4444444442014-04-04T04:04:042014-04-04T04:04:04
+2015-05-05 05:05:05.5555555552015-05-05 05:05:05.5555555552015-05-05T05:05:05.5555555552015-05-05T05:05:05.5555555552015-05-05T05:05:052015-05-05T05:05:05
+2016-06-06 06:06:06.6666666662016-06-06 06:06:06.6666666662016-06-06T06:06:06.6666666662016-06-06T06:06:06.6666666662016-06-06T06:06:062016-06-06T06:06:06
+2017-07-07 07:07:07.7777777772017-07-07 07:07:07.7777777772017-07-07T07:07:07.7777777772017-07-07T07:07:07.7777777772017-07-07T07:07:072017-07-07T07:07:07
+2018-08-08 08:08:08.8888888882018-08-08 08:08:08.8888888882018-08-08T08:08:08.8888888882018-08-08T08:08:08.8888888882018-08-08T08:08:082018-08-08T08:08:08
+2019-09-09 09:09:09.9999999992019-09-09 09:09:09.9999999992019-09-09T09:09:09.9999999992019-09-09T09:09:09.9999999992019-09-09T09:09:092019-09-09T09:09:09
+2020-10-10 10:10:10.1010101012020-10-10 10:10:10.1010101012020-10-10T10:10:10.1010101012020-10-10T10:10:10.1010101012020-10-10T10:10:102020-10-10T10:10:10
+2021-11-11 11:11:11.1111111112021-11-11 11:11:11.1111111112021-11-11T11:11:11.1111111112021-11-11T11:11:11.1111111112021-11-11T11:11:112021-11-11T11:11:11
+2022-12-12 12:12:12.1212121212022-12-12 12:12:12.1212121212022-12-12T12:12:12.1212121212022-12-12T12:12:12.1212121212022-12-12T12:12:122022-12-12T12:12:12
+2023-01-02 13:13:13.1313131312023-01-02 13:13:13.1313131312023-01-02T13:13:13.1313131312023-01-02T13:13:13.1313131312023-01-02T13:13:132023-01-02T13:13:13
+2024-02-02 14:14:14.1414141412024-02-02 14:14:14.1414141412024-02-02T14:14:14.1414141412024-02-02T14:14:14.1414141412024-02-02T14:14:142024-02-02T14:14:14
+2025-03-03 15:15:15.1515151512025-03-03 15:15:15.1515151512025-03-03T15:15:15.1515151512025-03-03T15:15:15.1515151512025-03-03T15:15:152025-03-03T15:15:15
+2026-04-04 16:16:16.1616161612026-04-04 16:16:16.1616161612026-04-04T16:16:16.1616161612026-04-04T16:16:16.1616161612026-04-04T16:16:162026-04-04T16:16:16
+2027-05-05 17:17:17.1717171712027-05-05 17:17:17.1717171712027-05-05T17:17:17.1717171712027-05-05T17:17:17.1717171712027-05-05T17:17:172027-05-05T17:17:17
+2028-06-06 18:18:18.1818181812028-06-06 18:18:18.1818181812028-06-06T18:18:18.1818181812028-06-06T18:18:18.1818181812028-06-06T18:18:182028-06-06T18:18:18
+2029-07-07 19:19:19.1919191912029-07-07 19:19:19.1919191912029-07-07T19:19:19.1919191912029-07-07T19:19:19.1919191912029-07-07T19:19:192029-07-07T19:19:19
+2030-08-08 20:20:20.2020202022030-08-08 20:20:20.2020202022030-08-08T20:20:20.2020202022030-08-08T20:20:20.2020202022030-08-08T20:20:202030-08-08T20:20:20
+2031-09-09 21:21:21.2121212122031-09-09 21:21:21.2121212122031-09-09T21:21:21.2121212122031-09-09T21:21:21.2121212122031-09-09T21:21:212031-09-09T21:21:21

Modified: hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java (original)
+++ hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java Mon Feb  2 17:04:41 2015
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.serde2.laz
 import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 
 public class DefaultHBaseKeyFactory extends AbstractHBaseKeyFactory implements HBaseKeyFactory {
@@ -44,8 +45,7 @@ public class DefaultHBaseKeyFactory exte
 
   @Override
   public ObjectInspector createKeyObjectInspector(TypeInfo type) throws SerDeException {
-    return LazyFactory.createLazyObjectInspector(type, serdeParams.getSeparators(), 1,
-        serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar());
+    return LazyFactory.createLazyObjectInspector(type, 1, serdeParams, ObjectInspectorOptions.JAVA);
   }
 
   @Override

Modified: hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseLazyObjectFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseLazyObjectFactory.java?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseLazyObjectFactory.java (original)
+++ hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseLazyObjectFactory.java Mon Feb  2 17:04:41 2015
@@ -25,8 +25,12 @@ import org.apache.hadoop.hive.hbase.stru
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
 import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.io.Text;
 
 // Does same thing with LazyFactory#createLazyObjectInspector except that this replaces
 // original keyOI with OI which is create by HBaseKeyFactory provided by serde property for hbase
@@ -46,8 +50,7 @@ public class HBaseLazyObjectFactory {
       }
     }
     return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(
-        serdeParams.getColumnNames(), columnObjectInspectors, serdeParams.getSeparators()[0],
-        serdeParams.getNullSequence(), serdeParams.isLastColumnTakesRest(),
-        serdeParams.isEscaped(), serdeParams.getEscapeChar());
+        serdeParams.getColumnNames(), columnObjectInspectors, null, serdeParams.getSeparators()[0],
+        serdeParams, ObjectInspectorOptions.JAVA);
   }
 }
\ No newline at end of file

Modified: hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/AvroHBaseValueFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/AvroHBaseValueFactory.java?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/AvroHBaseValueFactory.java (original)
+++ hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/AvroHBaseValueFactory.java Mon Feb  2 17:04:41 2015
@@ -74,9 +74,7 @@ public class AvroHBaseValueFactory exten
   @Override
   public ObjectInspector createValueObjectInspector(TypeInfo type) throws SerDeException {
     ObjectInspector oi =
-        LazyFactory.createLazyObjectInspector(type, serdeParams.getSeparators(), 1,
-        serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar(),
-        ObjectInspectorOptions.AVRO);
+        LazyFactory.createLazyObjectInspector(type, 1, serdeParams, ObjectInspectorOptions.AVRO);
 
     // initialize the object inspectors
     initInternalObjectInspectors(oi);

Modified: hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java (original)
+++ hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java Mon Feb  2 17:04:41 2015
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.serde2.laz
 import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
 import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 
@@ -62,8 +63,8 @@ public class DefaultHBaseValueFactory im
   @Override
   public ObjectInspector createValueObjectInspector(TypeInfo type)
       throws SerDeException {
-    return LazyFactory.createLazyObjectInspector(type, serdeParams.getSeparators(),
-        1, serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar());
+    return LazyFactory.createLazyObjectInspector(type,
+        1, serdeParams, ObjectInspectorOptions.JAVA);
   }
 
   @Override

Added: hive/trunk/hbase-handler/src/test/queries/positive/hbase_timestamp_format.q
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/queries/positive/hbase_timestamp_format.q?rev=1656519&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/queries/positive/hbase_timestamp_format.q (added)
+++ hive/trunk/hbase-handler/src/test/queries/positive/hbase_timestamp_format.q Mon Feb  2 17:04:41 2015
@@ -0,0 +1,21 @@
+
+create table hbase_str(rowkey string,mytime string,mystr string)
+  STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+  WITH SERDEPROPERTIES ('hbase.columns.mapping' = 'm:mytime,m:mystr')
+  TBLPROPERTIES ('hbase.table.name' = 'hbase_ts');
+
+describe hbase_str;
+insert overwrite table hbase_str select key, '2001-02-03-04.05.06.123456', value from src limit 3;
+select * from hbase_str;
+
+-- Timestamp string does not match the default timestamp format, specify a custom timestamp format
+create external table hbase_ts(rowkey string,mytime timestamp,mystr string)
+  STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+  WITH SERDEPROPERTIES ('hbase.columns.mapping' = 'm:mytime,m:mystr', 'timestamp.formats' = 'yyyy-MM-dd-HH.mm.ss.SSSSSS')
+  TBLPROPERTIES ('hbase.table.name' = 'hbase_ts');
+
+describe hbase_ts;
+select * from hbase_ts;
+
+drop table hbase_str;
+drop table hbase_ts;

Added: hive/trunk/hbase-handler/src/test/results/positive/hbase_timestamp_format.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/positive/hbase_timestamp_format.q.out?rev=1656519&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/positive/hbase_timestamp_format.q.out (added)
+++ hive/trunk/hbase-handler/src/test/results/positive/hbase_timestamp_format.q.out Mon Feb  2 17:04:41 2015
@@ -0,0 +1,94 @@
+PREHOOK: query: create table hbase_str(rowkey string,mytime string,mystr string)
+  STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+  WITH SERDEPROPERTIES ('hbase.columns.mapping' = 'm:mytime,m:mystr')
+  TBLPROPERTIES ('hbase.table.name' = 'hbase_ts')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hbase_str
+POSTHOOK: query: create table hbase_str(rowkey string,mytime string,mystr string)
+  STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+  WITH SERDEPROPERTIES ('hbase.columns.mapping' = 'm:mytime,m:mystr')
+  TBLPROPERTIES ('hbase.table.name' = 'hbase_ts')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_str
+PREHOOK: query: describe hbase_str
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@hbase_str
+POSTHOOK: query: describe hbase_str
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@hbase_str
+rowkey              	string              	from deserializer   
+mytime              	string              	from deserializer   
+mystr               	string              	from deserializer   
+PREHOOK: query: insert overwrite table hbase_str select key, '2001-02-03-04.05.06.123456', value from src limit 3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_str
+POSTHOOK: query: insert overwrite table hbase_str select key, '2001-02-03-04.05.06.123456', value from src limit 3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_str
+PREHOOK: query: select * from hbase_str
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_str
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_str
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_str
+#### A masked pattern was here ####
+238	2001-02-03-04.05.06.123456	val_238
+311	2001-02-03-04.05.06.123456	val_311
+86	2001-02-03-04.05.06.123456	val_86
+PREHOOK: query: -- Timestamp string does not match the default timestamp format, specify a custom timestamp format
+create external table hbase_ts(rowkey string,mytime timestamp,mystr string)
+  STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+  WITH SERDEPROPERTIES ('hbase.columns.mapping' = 'm:mytime,m:mystr', 'timestamp.formats' = 'yyyy-MM-dd-HH.mm.ss.SSSSSS')
+  TBLPROPERTIES ('hbase.table.name' = 'hbase_ts')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hbase_ts
+POSTHOOK: query: -- Timestamp string does not match the default timestamp format, specify a custom timestamp format
+create external table hbase_ts(rowkey string,mytime timestamp,mystr string)
+  STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+  WITH SERDEPROPERTIES ('hbase.columns.mapping' = 'm:mytime,m:mystr', 'timestamp.formats' = 'yyyy-MM-dd-HH.mm.ss.SSSSSS')
+  TBLPROPERTIES ('hbase.table.name' = 'hbase_ts')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_ts
+PREHOOK: query: describe hbase_ts
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@hbase_ts
+POSTHOOK: query: describe hbase_ts
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@hbase_ts
+rowkey              	string              	from deserializer   
+mytime              	timestamp           	from deserializer   
+mystr               	string              	from deserializer   
+PREHOOK: query: select * from hbase_ts
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_ts
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_ts
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_ts
+#### A masked pattern was here ####
+238	2001-02-03 04:05:06.123	val_238
+311	2001-02-03 04:05:06.123	val_311
+86	2001-02-03 04:05:06.123	val_86
+PREHOOK: query: drop table hbase_str
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hbase_str
+PREHOOK: Output: default@hbase_str
+POSTHOOK: query: drop table hbase_str
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hbase_str
+POSTHOOK: Output: default@hbase_str
+PREHOOK: query: drop table hbase_ts
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hbase_ts
+PREHOOK: Output: default@hbase_ts
+POSTHOOK: query: drop table hbase_ts
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hbase_ts
+POSTHOOK: Output: default@hbase_ts

Modified: hive/trunk/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/pom.xml?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/pom.xml (original)
+++ hive/trunk/pom.xml Mon Feb  2 17:04:41 2015
@@ -134,6 +134,7 @@
     <jersey.version>1.14</jersey.version>
     <jline.version>2.12</jline.version>
     <jms.version>1.1</jms.version>
+    <joda.version>2.5</joda.version>
     <jodd.version>3.5.2</jodd.version>
     <json.version>20090211</json.version>
     <junit.version>4.11</junit.version>

Modified: hive/trunk/ql/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/pom.xml?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/ql/pom.xml (original)
+++ hive/trunk/ql/pom.xml Mon Feb  2 17:04:41 2015
@@ -689,6 +689,7 @@
                   <include>com.google.guava:guava</include>
                   <include>net.sf.opencsv:opencsv</include>
                   <include>org.apache.hive:spark-client</include>
+                  <include>joda-time:joda-time</include>
                 </includes>
               </artifactSet>
               <relocations>

Added: hive/trunk/ql/src/test/queries/clientpositive/timestamp_formats.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/timestamp_formats.q?rev=1656519&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/timestamp_formats.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/timestamp_formats.q Mon Feb  2 17:04:41 2015
@@ -0,0 +1,23 @@
+
+CREATE TABLE timestamp_formats (
+  c1 string,
+  c1_ts timestamp,
+  c2 string,
+  c2_ts timestamp,
+  c3 string,
+  c3_ts timestamp
+);
+
+LOAD DATA LOCAL INPATH '../../data/files/ts_formats.txt' overwrite into table timestamp_formats;
+
+SELECT * FROM timestamp_formats;
+
+-- Add single timestamp format. This should allow c3_ts to parse
+ALTER TABLE timestamp_formats SET SERDEPROPERTIES ("timestamp.formats"="yyyy-MM-dd'T'HH:mm:ss");
+SELECT * FROM timestamp_formats;
+
+-- Add another format, to allow c2_ts to parse
+ALTER TABLE timestamp_formats SET SERDEPROPERTIES ("timestamp.formats"="yyyy-MM-dd'T'HH:mm:ss,yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS");
+SELECT * FROM timestamp_formats;
+
+DROP TABLE timestamp_formats;

Added: hive/trunk/ql/src/test/results/clientpositive/timestamp_formats.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/timestamp_formats.q.out?rev=1656519&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/timestamp_formats.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/timestamp_formats.q.out Mon Feb  2 17:04:41 2015
@@ -0,0 +1,145 @@
+PREHOOK: query: CREATE TABLE timestamp_formats (
+  c1 string,
+  c1_ts timestamp,
+  c2 string,
+  c2_ts timestamp,
+  c3 string,
+  c3_ts timestamp
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@timestamp_formats
+POSTHOOK: query: CREATE TABLE timestamp_formats (
+  c1 string,
+  c1_ts timestamp,
+  c2 string,
+  c2_ts timestamp,
+  c3 string,
+  c3_ts timestamp
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@timestamp_formats
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ts_formats.txt' overwrite into table timestamp_formats
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@timestamp_formats
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ts_formats.txt' overwrite into table timestamp_formats
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@timestamp_formats
+PREHOOK: query: SELECT * FROM timestamp_formats
+PREHOOK: type: QUERY
+PREHOOK: Input: default@timestamp_formats
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM timestamp_formats
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@timestamp_formats
+#### A masked pattern was here ####
+2011-01-01 01:01:01.111111111	2011-01-01 01:01:01.111111111	2011-01-01T01:01:01.111111111	NULL	2011-01-01T01:01:01	NULL
+2012-02-02 02:02:02.222222222	2012-02-02 02:02:02.222222222	2012-02-02T02:02:02.222222222	NULL	2012-02-02T02:02:02	NULL
+2013-03-03 03:03:03.333333333	2013-03-03 03:03:03.333333333	2013-03-03T03:03:03.333333333	NULL	2013-03-03T03:03:03	NULL
+2014-04-04 04:04:04.444444444	2014-04-04 04:04:04.444444444	2014-04-04T04:04:04.444444444	NULL	2014-04-04T04:04:04	NULL
+2015-05-05 05:05:05.555555555	2015-05-05 05:05:05.555555555	2015-05-05T05:05:05.555555555	NULL	2015-05-05T05:05:05	NULL
+2016-06-06 06:06:06.666666666	2016-06-06 06:06:06.666666666	2016-06-06T06:06:06.666666666	NULL	2016-06-06T06:06:06	NULL
+2017-07-07 07:07:07.777777777	2017-07-07 07:07:07.777777777	2017-07-07T07:07:07.777777777	NULL	2017-07-07T07:07:07	NULL
+2018-08-08 08:08:08.888888888	2018-08-08 08:08:08.888888888	2018-08-08T08:08:08.888888888	NULL	2018-08-08T08:08:08	NULL
+2019-09-09 09:09:09.999999999	2019-09-09 09:09:09.999999999	2019-09-09T09:09:09.999999999	NULL	2019-09-09T09:09:09	NULL
+2020-10-10 10:10:10.101010101	2020-10-10 10:10:10.101010101	2020-10-10T10:10:10.101010101	NULL	2020-10-10T10:10:10	NULL
+2021-11-11 11:11:11.111111111	2021-11-11 11:11:11.111111111	2021-11-11T11:11:11.111111111	NULL	2021-11-11T11:11:11	NULL
+2022-12-12 12:12:12.121212121	2022-12-12 12:12:12.121212121	2022-12-12T12:12:12.121212121	NULL	2022-12-12T12:12:12	NULL
+2023-01-02 13:13:13.131313131	2023-01-02 13:13:13.131313131	2023-01-02T13:13:13.131313131	NULL	2023-01-02T13:13:13	NULL
+2024-02-02 14:14:14.141414141	2024-02-02 14:14:14.141414141	2024-02-02T14:14:14.141414141	NULL	2024-02-02T14:14:14	NULL
+2025-03-03 15:15:15.151515151	2025-03-03 15:15:15.151515151	2025-03-03T15:15:15.151515151	NULL	2025-03-03T15:15:15	NULL
+2026-04-04 16:16:16.161616161	2026-04-04 16:16:16.161616161	2026-04-04T16:16:16.161616161	NULL	2026-04-04T16:16:16	NULL
+2027-05-05 17:17:17.171717171	2027-05-05 17:17:17.171717171	2027-05-05T17:17:17.171717171	NULL	2027-05-05T17:17:17	NULL
+2028-06-06 18:18:18.181818181	2028-06-06 18:18:18.181818181	2028-06-06T18:18:18.181818181	NULL	2028-06-06T18:18:18	NULL
+2029-07-07 19:19:19.191919191	2029-07-07 19:19:19.191919191	2029-07-07T19:19:19.191919191	NULL	2029-07-07T19:19:19	NULL
+2030-08-08 20:20:20.202020202	2030-08-08 20:20:20.202020202	2030-08-08T20:20:20.202020202	NULL	2030-08-08T20:20:20	NULL
+2031-09-09 21:21:21.212121212	2031-09-09 21:21:21.212121212	2031-09-09T21:21:21.212121212	NULL	2031-09-09T21:21:21	NULL
+PREHOOK: query: -- Add single timestamp format. This should allow c3_ts to parse
+ALTER TABLE timestamp_formats SET SERDEPROPERTIES ("timestamp.formats"="yyyy-MM-dd'T'HH:mm:ss")
+PREHOOK: type: ALTERTABLE_SERDEPROPERTIES
+PREHOOK: Input: default@timestamp_formats
+PREHOOK: Output: default@timestamp_formats
+POSTHOOK: query: -- Add single timestamp format. This should allow c3_ts to parse
+ALTER TABLE timestamp_formats SET SERDEPROPERTIES ("timestamp.formats"="yyyy-MM-dd'T'HH:mm:ss")
+POSTHOOK: type: ALTERTABLE_SERDEPROPERTIES
+POSTHOOK: Input: default@timestamp_formats
+POSTHOOK: Output: default@timestamp_formats
+PREHOOK: query: SELECT * FROM timestamp_formats
+PREHOOK: type: QUERY
+PREHOOK: Input: default@timestamp_formats
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM timestamp_formats
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@timestamp_formats
+#### A masked pattern was here ####
+2011-01-01 01:01:01.111111111	2011-01-01 01:01:01.111111111	2011-01-01T01:01:01.111111111	NULL	2011-01-01T01:01:01	2011-01-01 01:01:01
+2012-02-02 02:02:02.222222222	2012-02-02 02:02:02.222222222	2012-02-02T02:02:02.222222222	NULL	2012-02-02T02:02:02	2012-02-02 02:02:02
+2013-03-03 03:03:03.333333333	2013-03-03 03:03:03.333333333	2013-03-03T03:03:03.333333333	NULL	2013-03-03T03:03:03	2013-03-03 03:03:03
+2014-04-04 04:04:04.444444444	2014-04-04 04:04:04.444444444	2014-04-04T04:04:04.444444444	NULL	2014-04-04T04:04:04	2014-04-04 04:04:04
+2015-05-05 05:05:05.555555555	2015-05-05 05:05:05.555555555	2015-05-05T05:05:05.555555555	NULL	2015-05-05T05:05:05	2015-05-05 05:05:05
+2016-06-06 06:06:06.666666666	2016-06-06 06:06:06.666666666	2016-06-06T06:06:06.666666666	NULL	2016-06-06T06:06:06	2016-06-06 06:06:06
+2017-07-07 07:07:07.777777777	2017-07-07 07:07:07.777777777	2017-07-07T07:07:07.777777777	NULL	2017-07-07T07:07:07	2017-07-07 07:07:07
+2018-08-08 08:08:08.888888888	2018-08-08 08:08:08.888888888	2018-08-08T08:08:08.888888888	NULL	2018-08-08T08:08:08	2018-08-08 08:08:08
+2019-09-09 09:09:09.999999999	2019-09-09 09:09:09.999999999	2019-09-09T09:09:09.999999999	NULL	2019-09-09T09:09:09	2019-09-09 09:09:09
+2020-10-10 10:10:10.101010101	2020-10-10 10:10:10.101010101	2020-10-10T10:10:10.101010101	NULL	2020-10-10T10:10:10	2020-10-10 10:10:10
+2021-11-11 11:11:11.111111111	2021-11-11 11:11:11.111111111	2021-11-11T11:11:11.111111111	NULL	2021-11-11T11:11:11	2021-11-11 11:11:11
+2022-12-12 12:12:12.121212121	2022-12-12 12:12:12.121212121	2022-12-12T12:12:12.121212121	NULL	2022-12-12T12:12:12	2022-12-12 12:12:12
+2023-01-02 13:13:13.131313131	2023-01-02 13:13:13.131313131	2023-01-02T13:13:13.131313131	NULL	2023-01-02T13:13:13	2023-01-02 13:13:13
+2024-02-02 14:14:14.141414141	2024-02-02 14:14:14.141414141	2024-02-02T14:14:14.141414141	NULL	2024-02-02T14:14:14	2024-02-02 14:14:14
+2025-03-03 15:15:15.151515151	2025-03-03 15:15:15.151515151	2025-03-03T15:15:15.151515151	NULL	2025-03-03T15:15:15	2025-03-03 15:15:15
+2026-04-04 16:16:16.161616161	2026-04-04 16:16:16.161616161	2026-04-04T16:16:16.161616161	NULL	2026-04-04T16:16:16	2026-04-04 16:16:16
+2027-05-05 17:17:17.171717171	2027-05-05 17:17:17.171717171	2027-05-05T17:17:17.171717171	NULL	2027-05-05T17:17:17	2027-05-05 17:17:17
+2028-06-06 18:18:18.181818181	2028-06-06 18:18:18.181818181	2028-06-06T18:18:18.181818181	NULL	2028-06-06T18:18:18	2028-06-06 18:18:18
+2029-07-07 19:19:19.191919191	2029-07-07 19:19:19.191919191	2029-07-07T19:19:19.191919191	NULL	2029-07-07T19:19:19	2029-07-07 19:19:19
+2030-08-08 20:20:20.202020202	2030-08-08 20:20:20.202020202	2030-08-08T20:20:20.202020202	NULL	2030-08-08T20:20:20	2030-08-08 20:20:20
+2031-09-09 21:21:21.212121212	2031-09-09 21:21:21.212121212	2031-09-09T21:21:21.212121212	NULL	2031-09-09T21:21:21	2031-09-09 21:21:21
+PREHOOK: query: -- Add another format, to allow c2_ts to parse
+ALTER TABLE timestamp_formats SET SERDEPROPERTIES ("timestamp.formats"="yyyy-MM-dd'T'HH:mm:ss,yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS")
+PREHOOK: type: ALTERTABLE_SERDEPROPERTIES
+PREHOOK: Input: default@timestamp_formats
+PREHOOK: Output: default@timestamp_formats
+POSTHOOK: query: -- Add another format, to allow c2_ts to parse
+ALTER TABLE timestamp_formats SET SERDEPROPERTIES ("timestamp.formats"="yyyy-MM-dd'T'HH:mm:ss,yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS")
+POSTHOOK: type: ALTERTABLE_SERDEPROPERTIES
+POSTHOOK: Input: default@timestamp_formats
+POSTHOOK: Output: default@timestamp_formats
+PREHOOK: query: SELECT * FROM timestamp_formats
+PREHOOK: type: QUERY
+PREHOOK: Input: default@timestamp_formats
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM timestamp_formats
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@timestamp_formats
+#### A masked pattern was here ####
+2011-01-01 01:01:01.111111111	2011-01-01 01:01:01.111111111	2011-01-01T01:01:01.111111111	2011-01-01 01:01:01.111	2011-01-01T01:01:01	2011-01-01 01:01:01
+2012-02-02 02:02:02.222222222	2012-02-02 02:02:02.222222222	2012-02-02T02:02:02.222222222	2012-02-02 02:02:02.222	2012-02-02T02:02:02	2012-02-02 02:02:02
+2013-03-03 03:03:03.333333333	2013-03-03 03:03:03.333333333	2013-03-03T03:03:03.333333333	2013-03-03 03:03:03.333	2013-03-03T03:03:03	2013-03-03 03:03:03
+2014-04-04 04:04:04.444444444	2014-04-04 04:04:04.444444444	2014-04-04T04:04:04.444444444	2014-04-04 04:04:04.444	2014-04-04T04:04:04	2014-04-04 04:04:04
+2015-05-05 05:05:05.555555555	2015-05-05 05:05:05.555555555	2015-05-05T05:05:05.555555555	2015-05-05 05:05:05.555	2015-05-05T05:05:05	2015-05-05 05:05:05
+2016-06-06 06:06:06.666666666	2016-06-06 06:06:06.666666666	2016-06-06T06:06:06.666666666	2016-06-06 06:06:06.666	2016-06-06T06:06:06	2016-06-06 06:06:06
+2017-07-07 07:07:07.777777777	2017-07-07 07:07:07.777777777	2017-07-07T07:07:07.777777777	2017-07-07 07:07:07.777	2017-07-07T07:07:07	2017-07-07 07:07:07
+2018-08-08 08:08:08.888888888	2018-08-08 08:08:08.888888888	2018-08-08T08:08:08.888888888	2018-08-08 08:08:08.888	2018-08-08T08:08:08	2018-08-08 08:08:08
+2019-09-09 09:09:09.999999999	2019-09-09 09:09:09.999999999	2019-09-09T09:09:09.999999999	2019-09-09 09:09:09.999	2019-09-09T09:09:09	2019-09-09 09:09:09
+2020-10-10 10:10:10.101010101	2020-10-10 10:10:10.101010101	2020-10-10T10:10:10.101010101	2020-10-10 10:10:10.101	2020-10-10T10:10:10	2020-10-10 10:10:10
+2021-11-11 11:11:11.111111111	2021-11-11 11:11:11.111111111	2021-11-11T11:11:11.111111111	2021-11-11 11:11:11.111	2021-11-11T11:11:11	2021-11-11 11:11:11
+2022-12-12 12:12:12.121212121	2022-12-12 12:12:12.121212121	2022-12-12T12:12:12.121212121	2022-12-12 12:12:12.121	2022-12-12T12:12:12	2022-12-12 12:12:12
+2023-01-02 13:13:13.131313131	2023-01-02 13:13:13.131313131	2023-01-02T13:13:13.131313131	2023-01-02 13:13:13.131	2023-01-02T13:13:13	2023-01-02 13:13:13
+2024-02-02 14:14:14.141414141	2024-02-02 14:14:14.141414141	2024-02-02T14:14:14.141414141	2024-02-02 14:14:14.141	2024-02-02T14:14:14	2024-02-02 14:14:14
+2025-03-03 15:15:15.151515151	2025-03-03 15:15:15.151515151	2025-03-03T15:15:15.151515151	2025-03-03 15:15:15.151	2025-03-03T15:15:15	2025-03-03 15:15:15
+2026-04-04 16:16:16.161616161	2026-04-04 16:16:16.161616161	2026-04-04T16:16:16.161616161	2026-04-04 16:16:16.161	2026-04-04T16:16:16	2026-04-04 16:16:16
+2027-05-05 17:17:17.171717171	2027-05-05 17:17:17.171717171	2027-05-05T17:17:17.171717171	2027-05-05 17:17:17.171	2027-05-05T17:17:17	2027-05-05 17:17:17
+2028-06-06 18:18:18.181818181	2028-06-06 18:18:18.181818181	2028-06-06T18:18:18.181818181	2028-06-06 18:18:18.181	2028-06-06T18:18:18	2028-06-06 18:18:18
+2029-07-07 19:19:19.191919191	2029-07-07 19:19:19.191919191	2029-07-07T19:19:19.191919191	2029-07-07 19:19:19.191	2029-07-07T19:19:19	2029-07-07 19:19:19
+2030-08-08 20:20:20.202020202	2030-08-08 20:20:20.202020202	2030-08-08T20:20:20.202020202	2030-08-08 20:20:20.202	2030-08-08T20:20:20	2030-08-08 20:20:20
+2031-09-09 21:21:21.212121212	2031-09-09 21:21:21.212121212	2031-09-09T21:21:21.212121212	2031-09-09 21:21:21.212	2031-09-09T21:21:21	2031-09-09 21:21:21
+PREHOOK: query: DROP TABLE timestamp_formats
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@timestamp_formats
+PREHOOK: Output: default@timestamp_formats
+POSTHOOK: query: DROP TABLE timestamp_formats
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@timestamp_formats
+POSTHOOK: Output: default@timestamp_formats

Modified: hive/trunk/serde/if/serde.thrift
URL: http://svn.apache.org/viewvc/hive/trunk/serde/if/serde.thrift?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/serde/if/serde.thrift (original)
+++ hive/trunk/serde/if/serde.thrift Mon Feb  2 17:04:41 2015
@@ -69,6 +69,8 @@ const string UNION_TYPE_NAME  = "unionty
 const string LIST_COLUMNS = "columns";
 const string LIST_COLUMN_TYPES = "columns.types";
 
+const string TIMESTAMP_FORMATS = "timestamp.formats";
+
 const set<string> PrimitiveTypes  = [ VOID_TYPE_NAME BOOLEAN_TYPE_NAME TINYINT_TYPE_NAME SMALLINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME FLOAT_TYPE_NAME DOUBLE_TYPE_NAME STRING_TYPE_NAME  VARCHAR_TYPE_NAME CHAR_TYPE_NAME DATE_TYPE_NAME DATETIME_TYPE_NAME TIMESTAMP_TYPE_NAME DECIMAL_TYPE_NAME BINARY_TYPE_NAME],
 const set<string> CollectionTypes = [ LIST_TYPE_NAME MAP_TYPE_NAME ],
 const set<string> IntegralTypes = [ TINYINT_TYPE_NAME SMALLINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME ],

Modified: hive/trunk/serde/src/gen/thrift/gen-cpp/serde_constants.cpp
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/gen/thrift/gen-cpp/serde_constants.cpp?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/serde/src/gen/thrift/gen-cpp/serde_constants.cpp (original)
+++ hive/trunk/serde/src/gen/thrift/gen-cpp/serde_constants.cpp Mon Feb  2 17:04:41 2015
@@ -89,6 +89,8 @@ serdeConstants::serdeConstants() {
 
   LIST_COLUMN_TYPES = "columns.types";
 
+  TIMESTAMP_FORMATS = "timestamp.formats";
+
   PrimitiveTypes.insert("void");
   PrimitiveTypes.insert("boolean");
   PrimitiveTypes.insert("tinyint");

Modified: hive/trunk/serde/src/gen/thrift/gen-cpp/serde_constants.h
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/gen/thrift/gen-cpp/serde_constants.h?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/serde/src/gen/thrift/gen-cpp/serde_constants.h (original)
+++ hive/trunk/serde/src/gen/thrift/gen-cpp/serde_constants.h Mon Feb  2 17:04:41 2015
@@ -54,6 +54,7 @@ class serdeConstants {
   std::string UNION_TYPE_NAME;
   std::string LIST_COLUMNS;
   std::string LIST_COLUMN_TYPES;
+  std::string TIMESTAMP_FORMATS;
   std::set<std::string>  PrimitiveTypes;
   std::set<std::string>  CollectionTypes;
   std::set<std::string>  IntegralTypes;

Modified: hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java (original)
+++ hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java Mon Feb  2 17:04:41 2015
@@ -111,6 +111,8 @@ public class serdeConstants {
 
   public static final String LIST_COLUMN_TYPES = "columns.types";
 
+  public static final String TIMESTAMP_FORMATS = "timestamp.formats";
+
   public static final Set<String> PrimitiveTypes = new HashSet<String>();
   static {
     PrimitiveTypes.add("void");

Modified: hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/ThriftTestObj.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/ThriftTestObj.java?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/ThriftTestObj.java (original)
+++ hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/ThriftTestObj.java Mon Feb  2 17:04:41 2015
@@ -528,7 +528,7 @@ public class ThriftTestObj implements or
                 struct.field3 = new ArrayList<InnerStruct>(_list0.size);
                 for (int _i1 = 0; _i1 < _list0.size; ++_i1)
                 {
-                  InnerStruct _elem2; // required
+                  InnerStruct _elem2; // optional
                   _elem2 = new InnerStruct();
                   _elem2.read(iprot);
                   struct.field3.add(_elem2);
@@ -636,7 +636,7 @@ public class ThriftTestObj implements or
           struct.field3 = new ArrayList<InnerStruct>(_list5.size);
           for (int _i6 = 0; _i6 < _list5.size; ++_i6)
           {
-            InnerStruct _elem7; // required
+            InnerStruct _elem7; // optional
             _elem7 = new InnerStruct();
             _elem7.read(iprot);
             struct.field3.add(_elem7);

Modified: hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java (original)
+++ hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java Mon Feb  2 17:04:41 2015
@@ -1211,7 +1211,7 @@ public class Complex implements org.apac
                 struct.lint = new ArrayList<Integer>(_list18.size);
                 for (int _i19 = 0; _i19 < _list18.size; ++_i19)
                 {
-                  int _elem20; // required
+                  int _elem20; // optional
                   _elem20 = iprot.readI32();
                   struct.lint.add(_elem20);
                 }
@@ -1229,7 +1229,7 @@ public class Complex implements org.apac
                 struct.lString = new ArrayList<String>(_list21.size);
                 for (int _i22 = 0; _i22 < _list21.size; ++_i22)
                 {
-                  String _elem23; // required
+                  String _elem23; // optional
                   _elem23 = iprot.readString();
                   struct.lString.add(_elem23);
                 }
@@ -1247,7 +1247,7 @@ public class Complex implements org.apac
                 struct.lintString = new ArrayList<IntString>(_list24.size);
                 for (int _i25 = 0; _i25 < _list24.size; ++_i25)
                 {
-                  IntString _elem26; // required
+                  IntString _elem26; // optional
                   _elem26 = new IntString();
                   _elem26.read(iprot);
                   struct.lintString.add(_elem26);
@@ -1610,7 +1610,7 @@ public class Complex implements org.apac
           struct.lint = new ArrayList<Integer>(_list57.size);
           for (int _i58 = 0; _i58 < _list57.size; ++_i58)
           {
-            int _elem59; // required
+            int _elem59; // optional
             _elem59 = iprot.readI32();
             struct.lint.add(_elem59);
           }
@@ -1623,7 +1623,7 @@ public class Complex implements org.apac
           struct.lString = new ArrayList<String>(_list60.size);
           for (int _i61 = 0; _i61 < _list60.size; ++_i61)
           {
-            String _elem62; // required
+            String _elem62; // optional
             _elem62 = iprot.readString();
             struct.lString.add(_elem62);
           }
@@ -1636,7 +1636,7 @@ public class Complex implements org.apac
           struct.lintString = new ArrayList<IntString>(_list63.size);
           for (int _i64 = 0; _i64 < _list63.size; ++_i64)
           {
-            IntString _elem65; // required
+            IntString _elem65; // optional
             _elem65 = new IntString();
             _elem65.read(iprot);
             struct.lintString.add(_elem65);

Modified: hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MegaStruct.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MegaStruct.java?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MegaStruct.java (original)
+++ hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MegaStruct.java Mon Feb  2 17:04:41 2015
@@ -2280,7 +2280,7 @@ public class MegaStruct implements org.a
                     _val19 = new ArrayList<String>(_list20.size);
                     for (int _i21 = 0; _i21 < _list20.size; ++_i21)
                     {
-                      String _elem22; // required
+                      String _elem22; // optional
                       _elem22 = iprot.readString();
                       _val19.add(_elem22);
                     }
@@ -2310,7 +2310,7 @@ public class MegaStruct implements org.a
                     _val26 = new ArrayList<MiniStruct>(_list27.size);
                     for (int _i28 = 0; _i28 < _list27.size; ++_i28)
                     {
-                      MiniStruct _elem29; // required
+                      MiniStruct _elem29; // optional
                       _elem29 = new MiniStruct();
                       _elem29.read(iprot);
                       _val26.add(_elem29);
@@ -2333,7 +2333,7 @@ public class MegaStruct implements org.a
                 struct.my_stringlist = new ArrayList<String>(_list30.size);
                 for (int _i31 = 0; _i31 < _list30.size; ++_i31)
                 {
-                  String _elem32; // required
+                  String _elem32; // optional
                   _elem32 = iprot.readString();
                   struct.my_stringlist.add(_elem32);
                 }
@@ -2351,7 +2351,7 @@ public class MegaStruct implements org.a
                 struct.my_structlist = new ArrayList<MiniStruct>(_list33.size);
                 for (int _i34 = 0; _i34 < _list33.size; ++_i34)
                 {
-                  MiniStruct _elem35; // required
+                  MiniStruct _elem35; // optional
                   _elem35 = new MiniStruct();
                   _elem35.read(iprot);
                   struct.my_structlist.add(_elem35);
@@ -2370,7 +2370,7 @@ public class MegaStruct implements org.a
                 struct.my_enumlist = new ArrayList<MyEnum>(_list36.size);
                 for (int _i37 = 0; _i37 < _list36.size; ++_i37)
                 {
-                  MyEnum _elem38; // required
+                  MyEnum _elem38; // optional
                   _elem38 = MyEnum.findByValue(iprot.readI32());
                   struct.my_enumlist.add(_elem38);
                 }
@@ -2388,7 +2388,7 @@ public class MegaStruct implements org.a
                 struct.my_stringset = new HashSet<String>(2*_set39.size);
                 for (int _i40 = 0; _i40 < _set39.size; ++_i40)
                 {
-                  String _elem41; // required
+                  String _elem41; // optional
                   _elem41 = iprot.readString();
                   struct.my_stringset.add(_elem41);
                 }
@@ -2406,7 +2406,7 @@ public class MegaStruct implements org.a
                 struct.my_enumset = new HashSet<MyEnum>(2*_set42.size);
                 for (int _i43 = 0; _i43 < _set42.size; ++_i43)
                 {
-                  MyEnum _elem44; // required
+                  MyEnum _elem44; // optional
                   _elem44 = MyEnum.findByValue(iprot.readI32());
                   struct.my_enumset.add(_elem44);
                 }
@@ -2424,7 +2424,7 @@ public class MegaStruct implements org.a
                 struct.my_structset = new HashSet<MiniStruct>(2*_set45.size);
                 for (int _i46 = 0; _i46 < _set45.size; ++_i46)
                 {
-                  MiniStruct _elem47; // required
+                  MiniStruct _elem47; // optional
                   _elem47 = new MiniStruct();
                   _elem47.read(iprot);
                   struct.my_structset.add(_elem47);
@@ -3023,7 +3023,7 @@ public class MegaStruct implements org.a
               _val95 = new ArrayList<String>(_list96.size);
               for (int _i97 = 0; _i97 < _list96.size; ++_i97)
               {
-                String _elem98; // required
+                String _elem98; // optional
                 _elem98 = iprot.readString();
                 _val95.add(_elem98);
               }
@@ -3047,7 +3047,7 @@ public class MegaStruct implements org.a
               _val102 = new ArrayList<MiniStruct>(_list103.size);
               for (int _i104 = 0; _i104 < _list103.size; ++_i104)
               {
-                MiniStruct _elem105; // required
+                MiniStruct _elem105; // optional
                 _elem105 = new MiniStruct();
                 _elem105.read(iprot);
                 _val102.add(_elem105);
@@ -3064,7 +3064,7 @@ public class MegaStruct implements org.a
           struct.my_stringlist = new ArrayList<String>(_list106.size);
           for (int _i107 = 0; _i107 < _list106.size; ++_i107)
           {
-            String _elem108; // required
+            String _elem108; // optional
             _elem108 = iprot.readString();
             struct.my_stringlist.add(_elem108);
           }
@@ -3077,7 +3077,7 @@ public class MegaStruct implements org.a
           struct.my_structlist = new ArrayList<MiniStruct>(_list109.size);
           for (int _i110 = 0; _i110 < _list109.size; ++_i110)
           {
-            MiniStruct _elem111; // required
+            MiniStruct _elem111; // optional
             _elem111 = new MiniStruct();
             _elem111.read(iprot);
             struct.my_structlist.add(_elem111);
@@ -3091,7 +3091,7 @@ public class MegaStruct implements org.a
           struct.my_enumlist = new ArrayList<MyEnum>(_list112.size);
           for (int _i113 = 0; _i113 < _list112.size; ++_i113)
           {
-            MyEnum _elem114; // required
+            MyEnum _elem114; // optional
             _elem114 = MyEnum.findByValue(iprot.readI32());
             struct.my_enumlist.add(_elem114);
           }
@@ -3104,7 +3104,7 @@ public class MegaStruct implements org.a
           struct.my_stringset = new HashSet<String>(2*_set115.size);
           for (int _i116 = 0; _i116 < _set115.size; ++_i116)
           {
-            String _elem117; // required
+            String _elem117; // optional
             _elem117 = iprot.readString();
             struct.my_stringset.add(_elem117);
           }
@@ -3117,7 +3117,7 @@ public class MegaStruct implements org.a
           struct.my_enumset = new HashSet<MyEnum>(2*_set118.size);
           for (int _i119 = 0; _i119 < _set118.size; ++_i119)
           {
-            MyEnum _elem120; // required
+            MyEnum _elem120; // optional
             _elem120 = MyEnum.findByValue(iprot.readI32());
             struct.my_enumset.add(_elem120);
           }
@@ -3130,7 +3130,7 @@ public class MegaStruct implements org.a
           struct.my_structset = new HashSet<MiniStruct>(2*_set121.size);
           for (int _i122 = 0; _i122 < _set121.size; ++_i122)
           {
-            MiniStruct _elem123; // required
+            MiniStruct _elem123; // optional
             _elem123 = new MiniStruct();
             _elem123.read(iprot);
             struct.my_structset.add(_elem123);

Modified: hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/PropValueUnion.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/PropValueUnion.java?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/PropValueUnion.java (original)
+++ hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/PropValueUnion.java Mon Feb  2 17:04:41 2015
@@ -300,7 +300,7 @@ public class PropValueUnion extends org.
               lString = new ArrayList<String>(_list0.size);
               for (int _i1 = 0; _i1 < _list0.size; ++_i1)
               {
-                String _elem2; // required
+                String _elem2; // optional
                 _elem2 = iprot.readString();
                 lString.add(_elem2);
               }
@@ -423,7 +423,7 @@ public class PropValueUnion extends org.
             lString = new ArrayList<String>(_list9.size);
             for (int _i10 = 0; _i10 < _list9.size; ++_i10)
             {
-              String _elem11; // required
+              String _elem11; // optional
               _elem11 = iprot.readString();
               lString.add(_elem11);
             }

Modified: hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java (original)
+++ hive/trunk/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java Mon Feb  2 17:04:41 2015
@@ -431,7 +431,7 @@ public class SetIntString implements org
                 struct.sIntString = new HashSet<IntString>(2*_set82.size);
                 for (int _i83 = 0; _i83 < _set82.size; ++_i83)
                 {
-                  IntString _elem84; // required
+                  IntString _elem84; // optional
                   _elem84 = new IntString();
                   _elem84.read(iprot);
                   struct.sIntString.add(_elem84);
@@ -530,7 +530,7 @@ public class SetIntString implements org
           struct.sIntString = new HashSet<IntString>(2*_set87.size);
           for (int _i88 = 0; _i88 < _set87.size; ++_i88)
           {
-            IntString _elem89; // required
+            IntString _elem89; // optional
             _elem89 = new IntString();
             _elem89.read(iprot);
             struct.sIntString.add(_elem89);

Modified: hive/trunk/serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php (original)
+++ hive/trunk/serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php Mon Feb  2 17:04:41 2015
@@ -94,6 +94,8 @@ $GLOBALS['serde_CONSTANTS']['LIST_COLUMN
 
 $GLOBALS['serde_CONSTANTS']['LIST_COLUMN_TYPES'] = "columns.types";
 
+$GLOBALS['serde_CONSTANTS']['TIMESTAMP_FORMATS'] = "timestamp.formats";
+
 $GLOBALS['serde_CONSTANTS']['PrimitiveTypes'] = array(
   "void" => true,
   "boolean" => true,

Modified: hive/trunk/serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py (original)
+++ hive/trunk/serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py Mon Feb  2 17:04:41 2015
@@ -48,6 +48,7 @@ STRUCT_TYPE_NAME = "struct"
 UNION_TYPE_NAME = "uniontype"
 LIST_COLUMNS = "columns"
 LIST_COLUMN_TYPES = "columns.types"
+TIMESTAMP_FORMATS = "timestamp.formats"
 PrimitiveTypes = set([
   "void",
   "boolean",

Modified: hive/trunk/serde/src/gen/thrift/gen-rb/serde_constants.rb
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/gen/thrift/gen-rb/serde_constants.rb?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/serde/src/gen/thrift/gen-rb/serde_constants.rb (original)
+++ hive/trunk/serde/src/gen/thrift/gen-rb/serde_constants.rb Mon Feb  2 17:04:41 2015
@@ -85,6 +85,8 @@ LIST_COLUMNS = %q"columns"
 
 LIST_COLUMN_TYPES = %q"columns.types"
 
+TIMESTAMP_FORMATS = %q"timestamp.formats"
+
 PrimitiveTypes = Set.new([
   %q"void",
   %q"boolean",

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java Mon Feb  2 17:04:41 2015
@@ -44,6 +44,7 @@ import org.apache.hadoop.hive.serde2.laz
 import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
 import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
 import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyUnionObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters;
 import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -84,6 +85,7 @@ public class AvroLazyObjectInspector ext
    * @param escaped whether the data is escaped or not
    * @param escapeChar if escaped is true, the escape character
    * */
+  @Deprecated
   public AvroLazyObjectInspector(List<String> structFieldNames,
       List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments,
       byte separator, Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
@@ -92,6 +94,12 @@ public class AvroLazyObjectInspector ext
         nullSequence, lastColumnTakesRest, escaped, escapeChar);
   }
 
+  public AvroLazyObjectInspector(List<String> structFieldNames,
+      List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments,
+      byte separator, LazyObjectInspectorParameters lazyParams) {
+    super(structFieldNames, structFieldObjectInspectors, structFieldComments, separator, lazyParams);
+  }
+
   /**
    * Set the reader schema for the {@link AvroLazyObjectInspector} to the given schema
    * */

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java?rev=1656519&r1=1656518&r2=1656519&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java Mon Feb  2 17:04:41 2015
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.serde2.Ser
 import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
 import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
 import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
@@ -96,9 +97,7 @@ public class ColumnarSerDe extends Colum
     // Create the ObjectInspectors for the fields. Note: Currently
     // ColumnarObject uses same ObjectInpector as LazyStruct
     cachedObjectInspector = LazyFactory.createColumnarStructInspector(
-        serdeParams.getColumnNames(), serdeParams.getColumnTypes(), serdeParams
-            .getSeparators(), serdeParams.getNullSequence(), serdeParams
-            .isEscaped(), serdeParams.getEscapeChar());
+        serdeParams.getColumnNames(), serdeParams.getColumnTypes(), serdeParams);
 
     int size = serdeParams.getColumnTypes().size();
     List<Integer> notSkipIDs = new ArrayList<Integer>();



Mime
View raw message