hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r758070 - in /hadoop/hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/udf/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Date Tue, 24 Mar 2009 22:48:16 GMT
Author: namit
Date: Tue Mar 24 22:48:15 2009
New Revision: 758070

URL: http://svn.apache.org/viewvc?rev=758070&view=rev
Log:
HIVE-319. Add UDF for unix timestamp. 
(Hao Liu via namit)


Added:
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnixTimeStamp.java
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_unix_timestamp.q
    hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=758070&r1=758069&r2=758070&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Mar 24 22:48:15 2009
@@ -61,6 +61,9 @@
     HIVE-347. undo for Fix lot of partition scans in strict mode in case of error in 
     partition specification error. (namit)
 
+    HIVE-319. Add UDF for unix timestamp. 
+    (Hao Liu via namit)
+
 Release 0.2.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=758070&r1=758069&r2=758070&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Tue
Mar 24 22:48:15 2009
@@ -45,11 +45,11 @@
   static HashMap<String, FunctionInfo> mFunctions;
   static {
     mFunctions = new HashMap<String, FunctionInfo>();
-    registerUDF("default_sample_hashfn", UDFDefaultSampleHashFn.class, 
+    registerUDF("default_sample_hashfn", UDFDefaultSampleHashFn.class,
                 OperatorType.PREFIX, false);
     registerUDF("concat", UDFConcat.class, OperatorType.PREFIX, false);
     registerUDF("substr", UDFSubstr.class, OperatorType.PREFIX, false);
-    
+
     registerUDF("size", UDFSize.class, OperatorType.PREFIX, false);
 
     registerUDF("round", UDFRound.class, OperatorType.PREFIX, false);
@@ -66,7 +66,7 @@
     registerUDF("exp", UDFExp.class, OperatorType.PREFIX, false);
     registerUDF("power", UDFPower.class, OperatorType.PREFIX, false);
     registerUDF("pow", UDFPower.class, OperatorType.PREFIX, false);
-    
+
     registerUDF("upper", UDFUpper.class, OperatorType.PREFIX, false);
     registerUDF("lower", UDFLower.class, OperatorType.PREFIX, false);
     registerUDF("ucase", UDFUpper.class, OperatorType.PREFIX, false);
@@ -88,12 +88,13 @@
     registerUDF("month", UDFMonth.class, OperatorType.PREFIX, false);
     registerUDF("year", UDFYear.class, OperatorType.PREFIX, false);
     registerUDF("from_unixtime", UDFFromUnixTime.class, OperatorType.PREFIX, false);
+    registerUDF("unix_timestamp", UDFUnixTimeStamp.class, OperatorType.PREFIX, false);
     registerUDF("to_date", UDFDate.class, OperatorType.PREFIX, false);
 
     registerUDF("date_add", UDFDateAdd.class, OperatorType.PREFIX, false);
     registerUDF("date_sub", UDFDateSub.class, OperatorType.PREFIX, false);
     registerUDF("datediff", UDFDateDiff.class, OperatorType.PREFIX, false);
-    
+
     registerUDF("get_json_object", UDFJson.class, OperatorType.PREFIX, false);
 
     registerUDF("+", UDFOPPlus.class, OperatorType.INFIX, true);
@@ -126,7 +127,7 @@
     registerUDF("isnotnull", UDFOPNotNull.class, OperatorType.POSTFIX, true, "is not null");
 
     registerUDF("if", UDFIf.class, OperatorType.PREFIX, true);
-    
+
     // Aliases for Java Class Names
     // These are used in getImplicitConvertUDFMethod
     registerUDF(Boolean.class.getName(), UDFToBoolean.class, OperatorType.PREFIX, false,
@@ -153,7 +154,7 @@
     registerUDAF("count", UDAFCount.class);
     registerUDAF("max", UDAFMax.class);
     registerUDAF("min", UDAFMin.class);
-    registerUDAF("avg", UDAFAvg.class);    
+    registerUDAF("avg", UDAFAvg.class);
   }
 
   public static FunctionInfo getInfo(Class<?> fClass) {
@@ -181,7 +182,7 @@
 
   public static void registerUDF(String functionName, Class<? extends UDF> UDFClass,
                                  FunctionInfo.OperatorType opt, boolean isOperator) {
-    if (UDF.class.isAssignableFrom(UDFClass)) { 
+    if (UDF.class.isAssignableFrom(UDFClass)) {
       FunctionInfo fI = new FunctionInfo(functionName.toLowerCase(), UDFClass, null);
       fI.setIsOperator(isOperator);
       fI.setOpType(opt);
@@ -190,11 +191,11 @@
       throw new RuntimeException("Registering UDF Class " + UDFClass + " which does not extends
" + UDF.class);
     }
   }
-  
+
   public static void registerUDF(String functionName, Class<? extends UDF> UDFClass,
                                  FunctionInfo.OperatorType opt, boolean isOperator,
                                  String displayName) {
-    if (UDF.class.isAssignableFrom(UDFClass)) { 
+    if (UDF.class.isAssignableFrom(UDFClass)) {
       FunctionInfo fI = new FunctionInfo(displayName, UDFClass, null);
       fI.setIsOperator(isOperator);
       fI.setOpType(opt);
@@ -224,7 +225,7 @@
     numericTypes.put(Float.class, 5);
     numericTypes.put(Double.class, 6);
     numericTypes.put(String.class, 7);
-  } 
+  }
 
   /**
    * Find a common class that objects of both Class a and Class b can convert to.
@@ -263,7 +264,7 @@
     if (from.equals(Void.class)) {
       return true;
     }
-    
+
     // Allow implicit conversion from Byte -> Integer -> Long -> Float -> Double
-> String
     Integer f = numericTypes.get(from);
     Integer t = numericTypes.get(to);
@@ -273,7 +274,7 @@
   }
 
   /**
-   * Get the UDF method for the name and argumentClasses. 
+   * Get the UDF method for the name and argumentClasses.
    * @param name the name of the UDF
    * @param argumentClasses
    * @return
@@ -290,11 +291,11 @@
     catch (Exception e) {
       throw new RuntimeException("getUDFMethod exception: " + e.getMessage());
     }
-    return udfMethod;    
+    return udfMethod;
   }
 
   /**
-   * Get the UDAF evaluator for the name and argumentClasses. 
+   * Get the UDAF evaluator for the name and argumentClasses.
    * @param name the name of the UDAF
    * @param argumentClasses
    * @return
@@ -302,7 +303,7 @@
   public static Class<? extends UDAFEvaluator> getUDAFEvaluator(String name, List<Class<?>>
argumentClasses) {
     Class<? extends UDAF> udf = getUDAF(name);
     if (udf == null) return null;
-    
+
     Class<? extends UDAFEvaluator> evalClass = null;
     try {
       evalClass = udf.newInstance().getResolver().getEvaluatorClass(argumentClasses);
@@ -312,23 +313,23 @@
     catch (Exception e) {
       throw new RuntimeException("getUADFEvaluator exception: " + e.getMessage());
     }
-    return evalClass;    
+    return evalClass;
   }
 
   /**
    * This method is shared between UDFRegistry and UDAFRegistry.
-   * methodName will be "evaluate" for UDFRegistry, and "aggregate"/"evaluate"/"evaluatePartial"
for UDAFRegistry. 
+   * methodName will be "evaluate" for UDFRegistry, and "aggregate"/"evaluate"/"evaluatePartial"
for UDAFRegistry.
    */
   public static <T> Method getMethodInternal(Class<? extends T> udfClass, String
methodName, boolean exact, List<Class<?>> argumentClasses) {
 
     ArrayList<Method> mlist = new ArrayList<Method>();
-    
+
     for(Method m: Arrays.asList(udfClass.getMethods())) {
       if (m.getName().equals(methodName)) {
         mlist.add(m);
       }
     }
-    
+
     return getMethodInternal(mlist, exact, argumentClasses);
   }
 
@@ -373,7 +374,7 @@
   /**
    * Returns the evaluate method for the UDAF based on the aggregation mode.
    * See groupByDesc.Mode for details.
-   * 
+   *
    * @param name  name of the UDAF
    * @param mode  the mode of the aggregation
    * @return      null if no such UDAF is found
@@ -382,8 +383,8 @@
     Class<? extends UDAF> udaf = getUDAF(name);
     if (udaf == null)
       return null;
-    return FunctionRegistry.getMethodInternal(udaf, 
-        (mode == groupByDesc.Mode.COMPLETE || mode == groupByDesc.Mode.FINAL) 
+    return FunctionRegistry.getMethodInternal(udaf,
+        (mode == groupByDesc.Mode.COMPLETE || mode == groupByDesc.Mode.FINAL)
         ? "terminate" : "terminatePartial", true,
         new ArrayList<Class<?>>() );
   }
@@ -394,13 +395,13 @@
   public static Method getUDAFMethod(String name, Class<?>... argumentClasses) {
     return getUDAFMethod(name, Arrays.asList(argumentClasses));
   }
-  
+
   public static Object invoke(Method m, Object thisObject, Object[] arguments) throws HiveException
{
     Object o;
     try {
       o = m.invoke(thisObject, arguments);
     } catch (Exception e) {
-      String thisObjectString = "" + thisObject + " of class " + 
+      String thisObjectString = "" + thisObject + " of class " +
         (thisObject == null? "null" : thisObject.getClass().getName());
 
       StringBuilder argumentString = new StringBuilder();
@@ -420,11 +421,11 @@
         }
         argumentString.append("} of size " + arguments.length);
       }
-     
+
       e.printStackTrace();
-      throw new HiveException("Unable to execute method " + m + " " 
+      throw new HiveException("Unable to execute method " + m + " "
           + " on object " + thisObjectString
-          + " with arguments " + argumentString.toString() 
+          + " with arguments " + argumentString.toString()
           + ":" + e.getMessage());
     }
     return o;
@@ -432,7 +433,7 @@
 
   /**
    * Gets the closest matching method corresponding to the argument list from a list of methods.
-   * 
+   *
    * @param mlist The list of methods to inspect.
    * @param exact Boolean to indicate whether this is an exact match or not.
    * @param argumentClasses The classes for the argument.
@@ -469,7 +470,7 @@
           // Found an exact match
           if (leastImplicitConversions == 0) break;
         } else if (implicitConversions == leastImplicitConversions){
-          // Ambiguous call: two methods with the same number of implicit conversions 
+          // Ambiguous call: two methods with the same number of implicit conversions
           udfMethod = null;
         } else {
           // do nothing if implicitConversions > leastImplicitConversions

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnixTimeStamp.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnixTimeStamp.java?rev=758070&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnixTimeStamp.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnixTimeStamp.java Tue
Mar 24 22:48:15 2009
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf;
+
+import java.text.SimpleDateFormat;
+import java.text.ParseException;
+import java.util.TimeZone;
+import java.util.Date;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+
+@UDFType(deterministic=false)
+public class UDFUnixTimeStamp extends UDF {
+
+  private static Log LOG = LogFactory.getLog(UDFUnixTimeStamp.class.getName());
+
+  //  For now, we just use the default time zone.
+  private SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+
+  public UDFUnixTimeStamp() {
+  }
+
+  /**
+   * Return current UnixTime.
+   * @return long Number of seconds from 1970-01-01 00:00:00
+   */
+  public long evaluate()  {
+    Date date = new Date();
+    return date.getTime() / 1000;
+  }
+
+  /**
+   * Convert time string to UnixTime.
+   * @param datestring Time string in format yyyy-MM-dd HH:mm:ss
+   * @return long Number of seconds from 1970-01-01 00:00:00
+   */
+  public long evaluate(String datestring)  {
+    if (datestring == null) {
+      Date date = new Date();
+      return date.getTime() / 1000;
+    }
+
+    try {
+      Date date = (Date)formatter.parse(datestring);
+      return date.getTime() / 1000;
+    } catch (ParseException e) {
+      return 0;
+    }
+  }
+
+  /**
+   * Convert time string to UnixTime with user defined pattern.
+   * @param datestring Time string in format patternstring
+   * @param patternstring Time patterns string supported by SimpleDateFormat
+   * @return long Number of seconds from 1970-01-01 00:00:00
+   */
+  public long evaluate(String datestring, String patternstring)  {
+    try {
+      formatter.applyPattern(patternstring);
+    } catch (Exception e) {
+      return 0;
+    }
+
+    return evaluate(datestring);
+  }
+}

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_unix_timestamp.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_unix_timestamp.q?rev=758070&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_unix_timestamp.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_unix_timestamp.q Tue Mar 24 22:48:15
2009
@@ -0,0 +1,21 @@
+SELECT
+  '2009-03-20 11:30:01',
+  unix_timestamp('2009-03-20 11:30:01')
+FROM src LIMIT 1;
+
+SELECT
+  '2009-03-20',
+  unix_timestamp('2009-03-20', 'yyyy-MM-dd')
+FROM src LIMIT 1;
+
+SELECT
+  '2009 Mar 20 11:30:01 am',
+  unix_timestamp('2009 Mar 20 11:30:01 am', 'yyyy MMM dd h:mm:ss a')
+FROM src LIMIT 1;
+
+SELECT
+  'random_string',
+  unix_timestamp('random_string')
+FROM src LIMIT 1;
+
+

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out?rev=758070&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out Tue Mar
24 22:48:15 2009
@@ -0,0 +1,4 @@
+2009-03-20 11:30:01	1237573801
+2009-03-20	1237532400
+2009 Mar 20 11:30:01 am	1237573801
+random_string	0



Mime
View raw message