pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From the...@apache.org
Subject svn commit: r1376800 [1/6] - in /pig/trunk: ./ .eclipse.templates/ conf/ contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/ contrib/zebra/src/java/org/apache/hadoop/zebra/pig/ contrib/zebra/src/java/org/apache/hadoop/zebra/pig/compa...
Date Fri, 24 Aug 2012 00:18:09 GMT
Author: thejas
Date: Fri Aug 24 00:18:05 2012
New Revision: 1376800

URL: http://svn.apache.org/viewvc?rev=1376800&view=rev
Log:
PIG-1314: Add DateTime Support to Pig (zjshen via thejas)

Added:
    pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/DateTimeExpr.java
    pig/trunk/src/org/apache/pig/backend/hadoop/DateTimeWritable.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java.orig
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigDateTimeRawComparator.java
    pig/trunk/src/org/apache/pig/builtin/AddDuration.java
    pig/trunk/src/org/apache/pig/builtin/CurrentTime.java
    pig/trunk/src/org/apache/pig/builtin/DaysBetween.java
    pig/trunk/src/org/apache/pig/builtin/GetDay.java
    pig/trunk/src/org/apache/pig/builtin/GetHour.java
    pig/trunk/src/org/apache/pig/builtin/GetMilliSecond.java
    pig/trunk/src/org/apache/pig/builtin/GetMinute.java
    pig/trunk/src/org/apache/pig/builtin/GetMonth.java
    pig/trunk/src/org/apache/pig/builtin/GetSecond.java
    pig/trunk/src/org/apache/pig/builtin/GetWeek.java
    pig/trunk/src/org/apache/pig/builtin/GetWeekYear.java
    pig/trunk/src/org/apache/pig/builtin/GetYear.java
    pig/trunk/src/org/apache/pig/builtin/HoursBetween.java
    pig/trunk/src/org/apache/pig/builtin/MilliSecondsBetween.java
    pig/trunk/src/org/apache/pig/builtin/MinutesBetween.java
    pig/trunk/src/org/apache/pig/builtin/MonthsBetween.java
    pig/trunk/src/org/apache/pig/builtin/SecondsBetween.java
    pig/trunk/src/org/apache/pig/builtin/SubtractDuration.java
    pig/trunk/src/org/apache/pig/builtin/ToDate.java
    pig/trunk/src/org/apache/pig/builtin/ToDate2ARGS.java
    pig/trunk/src/org/apache/pig/builtin/ToDate3ARGS.java
    pig/trunk/src/org/apache/pig/builtin/ToDateISO.java
    pig/trunk/src/org/apache/pig/builtin/ToMilliSeconds.java
    pig/trunk/src/org/apache/pig/builtin/ToString.java
    pig/trunk/src/org/apache/pig/builtin/ToUnixTime.java
    pig/trunk/src/org/apache/pig/builtin/WeeksBetween.java
    pig/trunk/src/org/apache/pig/builtin/YearsBetween.java
    pig/trunk/src/org/apache/pig/impl/io/NullableDateTimeWritable.java
Modified:
    pig/trunk/.eclipse.templates/.classpath
    pig/trunk/CHANGES.txt
    pig/trunk/conf/pig.properties
    pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java
    pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java
    pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java
    pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/ExprUtils.java
    pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java
    pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt
    pig/trunk/src/org/apache/pig/LoadCaster.java
    pig/trunk/src/org/apache/pig/PigWarning.java
    pig/trunk/src/org/apache/pig/StoreCaster.java
    pig/trunk/src/org/apache/pig/backend/hadoop/HDataType.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ComparisonOperator.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POMapLookUp.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java
    pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java
    pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java
    pig/trunk/src/org/apache/pig/builtin/BinStorage.java
    pig/trunk/src/org/apache/pig/builtin/TextLoader.java
    pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java
    pig/trunk/src/org/apache/pig/data/BinInterSedes.java
    pig/trunk/src/org/apache/pig/data/DataReaderWriter.java
    pig/trunk/src/org/apache/pig/data/DataType.java
    pig/trunk/src/org/apache/pig/data/DefaultTuple.java
    pig/trunk/src/org/apache/pig/data/SchemaTuple.java
    pig/trunk/src/org/apache/pig/data/SchemaTupleClassGenerator.java
    pig/trunk/src/org/apache/pig/data/SizeUtil.java
    pig/trunk/src/org/apache/pig/data/TypeAwareTuple.java
    pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
    pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java
    pig/trunk/src/org/apache/pig/impl/util/CastUtils.java
    pig/trunk/src/org/apache/pig/impl/util/NumValCarrier.java
    pig/trunk/src/org/apache/pig/impl/util/StorageUtil.java
    pig/trunk/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java
    pig/trunk/src/org/apache/pig/parser/AliasMasker.g
    pig/trunk/src/org/apache/pig/parser/AstPrinter.g
    pig/trunk/src/org/apache/pig/parser/AstValidator.g
    pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g
    pig/trunk/src/org/apache/pig/parser/QueryLexer.g
    pig/trunk/src/org/apache/pig/parser/QueryParser.g
    pig/trunk/src/org/apache/pig/pen/AugmentBaseDataVisitor.java
    pig/trunk/src/org/apache/pig/scripting/jruby/JrubyScriptEngine.java
    pig/trunk/src/org/apache/pig/scripting/jruby/RubySchema.java
    pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java
    pig/trunk/test/org/apache/pig/data/TestSchemaTuple.java
    pig/trunk/test/org/apache/pig/test/TestAdd.java
    pig/trunk/test/org/apache/pig/test/TestBuiltin.java
    pig/trunk/test/org/apache/pig/test/TestConversions.java
    pig/trunk/test/org/apache/pig/test/TestDivide.java
    pig/trunk/test/org/apache/pig/test/TestEqualTo.java
    pig/trunk/test/org/apache/pig/test/TestGTOrEqual.java
    pig/trunk/test/org/apache/pig/test/TestGreaterThan.java
    pig/trunk/test/org/apache/pig/test/TestLTOrEqual.java
    pig/trunk/test/org/apache/pig/test/TestLessThan.java
    pig/trunk/test/org/apache/pig/test/TestMod.java
    pig/trunk/test/org/apache/pig/test/TestMultiply.java
    pig/trunk/test/org/apache/pig/test/TestNotEqualTo.java
    pig/trunk/test/org/apache/pig/test/TestNull.java
    pig/trunk/test/org/apache/pig/test/TestOrderBy.java
    pig/trunk/test/org/apache/pig/test/TestPOBinCond.java
    pig/trunk/test/org/apache/pig/test/TestPOCast.java
    pig/trunk/test/org/apache/pig/test/TestPackage.java
    pig/trunk/test/org/apache/pig/test/TestPigTupleRawComparator.java
    pig/trunk/test/org/apache/pig/test/TestResourceSchema.java
    pig/trunk/test/org/apache/pig/test/TestSchema.java
    pig/trunk/test/org/apache/pig/test/TestStore.java
    pig/trunk/test/org/apache/pig/test/TestSubtract.java
    pig/trunk/test/org/apache/pig/test/TestTextDataParser.java
    pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
    pig/trunk/test/org/apache/pig/test/utils/GenRandomData.java

Modified: pig/trunk/.eclipse.templates/.classpath
URL: http://svn.apache.org/viewvc/pig/trunk/.eclipse.templates/.classpath?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/.eclipse.templates/.classpath (original)
+++ pig/trunk/.eclipse.templates/.classpath Fri Aug 24 00:18:05 2012
@@ -25,7 +25,7 @@
 	<classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/js-1.7R2.jar"/>
 	<classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/zookeeper-3.3.3.jar"/>
 	<classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/guava-r06.jar"/>
-	<classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/jython-2.5.0.jar"/>
+	<classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/jython-2.5.2.jar"/>
 	<classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/hadoop-core-0.20.2.jar"/>
 	<classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/commons-logging-1.1.1.jar"/>
 	<classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/commons-cli-1.2.jar"/>

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Aug 24 00:18:05 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
 
 IMPROVEMENTS
 
+PIG-1314: Add DateTime Support to Pig (zjshen via thejas)
+
 PIG-2785: NoClassDefFoundError after upgrading to pig 0.10.0 from 0.9.0 (matterhayes via
sms)
 
 PIG-2556: CSVExcelStorage load: quoted field with newline as first character sees newline
as record end (tivv via dvryaboy)

Modified: pig/trunk/conf/pig.properties
URL: http://svn.apache.org/viewvc/pig/trunk/conf/pig.properties?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/conf/pig.properties (original)
+++ pig/trunk/conf/pig.properties Fri Aug 24 00:18:05 2012
@@ -17,6 +17,10 @@
 #exectype local|mapreduce, mapreduce is default
 #exectype=local
 
+#the default timezone: if it is not set, the default timezone for this host is used.
+#the correct timezone format is the UTC offset: e.g., +08:00. 
+#pig.datetime.default.tz=
+
 #pig.logfile=
 
 #Do not spill temp files smaller than this size (bytes)

Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java
(original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java
Fri Aug 24 00:18:05 2012
@@ -17,6 +17,8 @@
  */
 package org.apache.pig.piggybank.storage;
 
+import org.joda.time.DateTime;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.NullWritable;
@@ -115,6 +117,11 @@ public class DBStorage extends StoreFunc
             sqlPos++;
             break;
 
+          case DataType.DATETIME:
+            ps.setDate(sqlPos, ((DateTime) field).toDate());
+            sqlPos++;
+            break;
+
           case DataType.BYTEARRAY:
             byte[] b = ((DataByteArray) field).get();
             ps.setBytes(sqlPos, b);

Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java
(original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java
Fri Aug 24 00:18:05 2012
@@ -21,6 +21,8 @@ import java.io.IOException;
 import java.lang.reflect.Type;
 import java.util.ArrayList;
 
+import org.joda.time.DateTime;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.BooleanWritable;
@@ -40,6 +42,7 @@ import org.apache.hadoop.mapreduce.lib.i
 import org.apache.hadoop.mapreduce.lib.input.SequenceFileRecordReader;
 import org.apache.pig.FileInputLoadFunc;
 import org.apache.pig.backend.BackendException;
+import org.apache.pig.backend.hadoop.DateTimeWritable;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
 import org.apache.pig.data.DataByteArray;
 import org.apache.pig.data.DataType;
@@ -96,6 +99,7 @@ public class SequenceFileLoader extends 
     else if (t == DoubleWritable.class) return DataType.DOUBLE;
     else if (t == BooleanWritable.class) return DataType.BOOLEAN;
     else if (t == ByteWritable.class) return DataType.BYTE;
+    else if (t == DateTimeWritable.class) return DataType.DATETIME;
     // not doing maps or other complex types for now
     else return DataType.ERROR;
   }
@@ -110,6 +114,7 @@ public class SequenceFileLoader extends 
       case DataType.FLOAT: return ((FloatWritable) w).get();
       case DataType.DOUBLE: return ((DoubleWritable) w).get();
       case DataType.BYTE: return ((ByteWritable) w).get();
+      case DataType.DATETIME: return ((DateTimeWritable) w).get();
     }
     
     return null;

Modified: pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java (original)
+++ pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java Fri
Aug 24 00:18:05 2012
@@ -49,6 +49,9 @@ class SchemaConverter {
         case DataType.BOOLEAN:
             ret = ColumnType.BOOL; 
             break;
+        case DataType.DATETIME:
+            ret = ColumnType.DATETIME; 
+            break;
         case DataType.BAG:
             ret = ColumnType.COLLECTION; 
             break;

Added: pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/DateTimeExpr.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/DateTimeExpr.java?rev=1376800&view=auto
==============================================================================
--- pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/DateTimeExpr.java
(added)
+++ pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/DateTimeExpr.java
Fri Aug 24 00:18:05 2012
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.pig.comparator;
+
+import org.joda.time.DateTime;
+
+
+public final class DateTimeExpr extends FixedLengthPrimitive {
+  private static final int ONE_MINUTE = 60000;
+  public DateTimeExpr(int index) {
+    super(index, (Long.SIZE + Short.SIZE) / Byte.SIZE);
+  }
+
+  protected void convertValue(byte[] b, Object o) {
+    if (b.length != (Long.SIZE + Short.SIZE) / Byte.SIZE) {
+      throw new IllegalArgumentException("Incorrect buffer size");
+    }
+    long dt = ((DateTime) o).getMillis();
+    dt ^= 1L << (Long.SIZE - 1);
+    b[7] = (byte) dt;
+    b[6] = (byte) (dt >> 8);
+    b[5] = (byte) (dt >> 16);
+    b[4] = (byte) (dt >> 24);
+    b[3] = (byte) (dt >> 32);
+    b[2] = (byte) (dt >> 40);
+    b[1] = (byte) (dt >> 48);
+    b[0] = (byte) (dt >> 56);
+    
+    short dtz = (short) (((DateTime) o).getZone().getOffset((DateTime) o) / ONE_MINUTE);
+    dtz ^= 1 << (Short.SIZE - 1);
+    b[9] = (byte) dtz;
+    b[8] = (byte) (dtz >> 8);
+  }
+
+  @Override
+  protected String getType() {
+    return "DateTime";
+  }
+}

Modified: pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/ExprUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/ExprUtils.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/ExprUtils.java
(original)
+++ pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/ExprUtils.java
Fri Aug 24 00:18:05 2012
@@ -109,6 +109,8 @@ public class ExprUtils {
         return new IntExpr(index);
       case DataType.LONG:
         return new LongExpr(index);
+      case DataType.DATETIME:
+        return new DateTimeExpr(index);
       default:
         throw new RuntimeException("Not a prmitive PIG type");
     }

Modified: pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java (original)
+++ pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java Fri Aug
24 00:18:05 2012
@@ -79,6 +79,14 @@ public enum ColumnType {
     }
   },
   /**
+   * DateTime
+   */
+  DATETIME("datetime") {
+    public byte pigDataType() {
+      return DataType.DATETIME;
+    }
+  },
+  /**
    * Collection
    */
   COLLECTION("collection") {

Modified: pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt (original)
+++ pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt Fri Aug
24 00:18:05 2012
@@ -57,6 +57,7 @@ TOKEN : { <BOOL : "boolean"> }
 TOKEN : { <LONG : "long"> }
 TOKEN : { <FLOAT : "float"> }
 TOKEN : { <DOUBLE : "double"> }
+TOKEN : { <DATETIME : "datetime"> }
 TOKEN : { <STRING : "string"> }
 TOKEN : { <BYTES : "bytes"> }
 TOKEN : { <COLLECTION : "collection"> }

Modified: pig/trunk/src/org/apache/pig/LoadCaster.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/LoadCaster.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/LoadCaster.java (original)
+++ pig/trunk/src/org/apache/pig/LoadCaster.java Fri Aug 24 00:18:05 2012
@@ -19,6 +19,8 @@ package org.apache.pig;
 import java.io.IOException;
 import java.util.Map;
 
+import org.joda.time.DateTime;
+
 import org.apache.pig.ResourceSchema.ResourceFieldSchema;
 import org.apache.pig.classification.InterfaceAudience;
 import org.apache.pig.classification.InterfaceStability;
@@ -70,6 +72,14 @@ public interface LoadCaster {
     public Double bytesToDouble(byte[] b) throws IOException;
 
     /**
+     * Cast data from bytearray to datetime value.  
+     * @param b bytearray to be cast.
+     * @return datetime value.
+     * @throws IOException if the value cannot be cast.
+     */
+    public DateTime bytesToDateTime(byte[] b) throws IOException;
+
+    /**
      * Cast data from bytearray to integer value.  
      * @param b bytearray to be cast.
      * @return Double value.

Modified: pig/trunk/src/org/apache/pig/PigWarning.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigWarning.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigWarning.java (original)
+++ pig/trunk/src/org/apache/pig/PigWarning.java Fri Aug 24 00:18:05 2012
@@ -34,6 +34,7 @@ public enum PigWarning {
     IMPLICIT_CAST_TO_INT,
     IMPLICIT_CAST_TO_LONG,
     IMPLICIT_CAST_TO_BOOLEAN,
+    IMPLICIT_CAST_TO_DATETIME,
     IMPLICIT_CAST_TO_MAP,
     IMPLICIT_CAST_TO_TUPLE,
     TOO_LARGE_FOR_INT,

Modified: pig/trunk/src/org/apache/pig/StoreCaster.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/StoreCaster.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/StoreCaster.java (original)
+++ pig/trunk/src/org/apache/pig/StoreCaster.java Fri Aug 24 00:18:05 2012
@@ -20,6 +20,8 @@ package org.apache.pig;
 import java.io.IOException;
 import java.util.Map;
 
+import org.joda.time.DateTime;
+
 import org.apache.pig.classification.InterfaceAudience;
 import org.apache.pig.classification.InterfaceStability;
 import org.apache.pig.data.DataBag;
@@ -48,6 +50,8 @@ public interface StoreCaster extends Loa
 
     public byte[] toBytes(Long l) throws IOException;
 
+    public byte[] toBytes(DateTime dt) throws IOException;
+
     public byte[] toBytes(Map<String, Object> m) throws IOException;
 
     public byte[] toBytes(Tuple t) throws IOException;

Added: pig/trunk/src/org/apache/pig/backend/hadoop/DateTimeWritable.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/DateTimeWritable.java?rev=1376800&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/DateTimeWritable.java (added)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/DateTimeWritable.java Fri Aug 24 00:18:05
2012
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.backend.hadoop;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
+
+/**
+ * Writable for Double values.
+ */
+public class DateTimeWritable implements WritableComparable {
+    
+    private static final int ONE_MINUTE = 60000;
+
+    private DateTime value = null;
+
+    public DateTimeWritable() {
+
+    }
+
+    public DateTimeWritable(DateTime dt) {
+        value = dt;
+    }
+
+    public void readFields(DataInput in) throws IOException {
+        value = new DateTime(in.readLong(), DateTimeZone.forOffsetMillis(in.readShort() *
ONE_MINUTE));
+    }
+
+    public void write(DataOutput out) throws IOException {
+        out.writeLong(value.getMillis());
+        out.writeShort(value.getZone().getOffset(value) / ONE_MINUTE);
+    }
+
+    public void set(DateTime dt) {
+        value = dt;
+    }
+
+    public DateTime get() {
+        return value;
+    }
+
+    /**
+     * Returns true iff <code>o</code> is a DateTimeWritable with the same
+     * value.
+     */
+    @Override
+    public boolean equals(Object o) {
+        if (!(o instanceof DateTimeWritable)) {
+            return false;
+        }
+        DateTimeWritable other = (DateTimeWritable) o;
+        return this.value.equals(other.value);
+    }
+
+    @Override
+    public int hashCode() {
+        return value.hashCode();
+    }
+
+    public int compareTo(Object o) {
+        DateTimeWritable other = (DateTimeWritable) o;
+        return value.compareTo(other.value);
+    }
+
+    @Override
+    public String toString() {
+        return value.toString();
+    }
+
+    /** 
+     * A Comparator optimized for DateTimeWritable.
+     */
+    public static class Comparator extends WritableComparator {
+        public Comparator() {
+            super(DateTimeWritable.class);
+        }
+
+        @Override
+        public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+            DateTime thisValue = new DateTime(readLong(b1, s1));
+            DateTime thatValue = new DateTime(readLong(b2, s2));
+            return thisValue.compareTo(thatValue);
+        }
+    }
+
+    static { // register this comparator
+        WritableComparator.define(DateTimeWritable.class, new Comparator());
+    }
+}

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/HDataType.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/HDataType.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/HDataType.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/HDataType.java Fri Aug 24 00:18:05 2012
@@ -19,6 +19,8 @@ package org.apache.pig.backend.hadoop;
 
 import java.util.Map;
 
+import org.joda.time.DateTime;
+
 import org.apache.pig.PigException;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.data.DataBag;
@@ -32,6 +34,7 @@ import org.apache.pig.impl.io.NullableDo
 import org.apache.pig.impl.io.NullableFloatWritable;
 import org.apache.pig.impl.io.NullableIntWritable;
 import org.apache.pig.impl.io.NullableLongWritable;
+import org.apache.pig.impl.io.NullableDateTimeWritable;
 import org.apache.pig.impl.io.NullableText;
 import org.apache.pig.impl.io.NullableTuple;
 import org.apache.pig.impl.io.PigNullableWritable;
@@ -48,6 +51,7 @@ public class HDataType {
     static NullableDoubleWritable doubleWrit = new NullableDoubleWritable();
     static NullableIntWritable intWrit = new NullableIntWritable();
     static NullableLongWritable longWrit = new NullableLongWritable();
+    static NullableDateTimeWritable dtWrit = new NullableDateTimeWritable();
     static NullableBag defDB = new NullableBag();
     static NullableTuple defTup = new NullableTuple();
     static Map<Byte, String> typeToName = null;
@@ -82,7 +86,10 @@ public class HDataType {
            
         case DataType.LONG:
             return new NullableLongWritable((Long)o);
-          
+
+        case DataType.DATETIME:
+            return new NullableDateTimeWritable((DateTime)o);
+
         case DataType.TUPLE:
             return new NullableTuple((Tuple)o);
          
@@ -126,6 +133,10 @@ public class HDataType {
                 NullableLongWritable nLongWrit = new NullableLongWritable();
                 nLongWrit.setNull(true);
                 return nLongWrit;
+            case DataType.DATETIME:
+                NullableDateTimeWritable nDateTimeWrit = new NullableDateTimeWritable();
+                nDateTimeWrit.setNull(true);
+                return nDateTimeWrit;
             case DataType.TUPLE:
                 NullableTuple ntuple = new NullableTuple();
                 ntuple.setNull(true);
@@ -178,6 +189,9 @@ public class HDataType {
         case DataType.LONG:
             wcKey = longWrit;
             break;
+        case DataType.DATETIME:
+            wcKey = dtWrit;
+            break;
         case DataType.TUPLE:
             wcKey = defTup;
             break;
@@ -212,6 +226,8 @@ public class HDataType {
             return DataType.INTEGER;
         else if (o instanceof NullableLongWritable)
             return DataType.LONG;
+        else if (o instanceof NullableDateTimeWritable)
+            return DataType.DATETIME;
         else if (o instanceof NullableBag)
             return DataType.BAG;
         else if (o instanceof NullableTuple)

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Fri Aug 24 00:18:05 2012
@@ -84,6 +84,7 @@ import org.apache.pig.impl.io.NullableDo
 import org.apache.pig.impl.io.NullableFloatWritable;
 import org.apache.pig.impl.io.NullableIntWritable;
 import org.apache.pig.impl.io.NullableLongWritable;
+import org.apache.pig.impl.io.NullableDateTimeWritable;
 import org.apache.pig.impl.io.NullablePartitionWritable;
 import org.apache.pig.impl.io.NullableText;
 import org.apache.pig.impl.io.NullableTuple;
@@ -956,6 +957,12 @@ public class JobControlCompiler{
         }
     }
 
+    public static class PigDateTimeWritableComparator extends PigWritableComparator {
+        public PigDateTimeWritableComparator() {
+            super(NullableDateTimeWritable.class);
+        }
+    }
+
     public static class PigCharArrayWritableComparator extends PigWritableComparator {
         public PigCharArrayWritableComparator() {
             super(NullableText.class);
@@ -1012,6 +1019,12 @@ public class JobControlCompiler{
         }
     }
 
+    public static class PigGroupingDateTimeWritableComparator extends WritableComparator
{
+        public PigGroupingDateTimeWritableComparator() {
+            super(NullableDateTimeWritable.class, true);
+        }
+    }
+
     public static class PigGroupingCharArrayWritableComparator extends WritableComparator
{
         public PigGroupingCharArrayWritableComparator() {
             super(NullableText.class, true);
@@ -1087,6 +1100,10 @@ public class JobControlCompiler{
                 job.setSortComparatorClass(PigDoubleRawComparator.class);
                 break;
 
+            case DataType.DATETIME:
+                job.setSortComparatorClass(PigDateTimeRawComparator.class);
+                break;
+
             case DataType.CHARARRAY:
                 job.setSortComparatorClass(PigTextRawComparator.class);
                 break;
@@ -1141,6 +1158,11 @@ public class JobControlCompiler{
             job.setGroupingComparatorClass(PigGroupingDoubleWritableComparator.class);
             break;
 
+        case DataType.DATETIME:
+            job.setSortComparatorClass(PigDateTimeWritableComparator.class);
+            job.setGroupingComparatorClass(PigGroupingDateTimeWritableComparator.class);
+            break;
+
         case DataType.CHARARRAY:
             job.setSortComparatorClass(PigCharArrayWritableComparator.class);
             job.setGroupingComparatorClass(PigGroupingCharArrayWritableComparator.class);



Mime
View raw message