hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1476646 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/exec/ test/queries/clientpositive/ test/results/clientpositive/
Date Sat, 27 Apr 2013 17:53:49 GMT
Author: hashutosh
Date: Sat Apr 27 17:53:48 2013
New Revision: 1476646

URL: http://svn.apache.org/r1476646
Log:
HIVE-4019 : Ability to create and drop temporary partition function (Brock Noland via Ashutosh
Chauhan)

Added:
    hive/trunk/ql/src/test/queries/clientpositive/ptf_register_tblfn.q
    hive/trunk/ql/src/test/results/clientpositive/ptf_register_tblfn.q.out
Removed:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFFunctionInfo.java
Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
    hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java?rev=1476646&r1=1476645&r2=1476646&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java Sat Apr 27 17:53:48
2013
@@ -23,6 +23,8 @@ import org.apache.hadoop.hive.ql.udf.gen
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
+import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver;
+import org.apache.hadoop.hive.ql.udf.ptf.WindowingTableFunction;
 
 /**
  * FunctionInfo.
@@ -32,6 +34,8 @@ public class FunctionInfo {
 
   private final boolean isNative;
 
+  private final boolean isInternalTableFunction;
+
   private final String displayName;
 
   private GenericUDF genericUDF;
@@ -40,11 +44,14 @@ public class FunctionInfo {
 
   private GenericUDAFResolver genericUDAFResolver;
 
+  private Class<? extends TableFunctionResolver>  tableFunctionResolver;
+
   public FunctionInfo(boolean isNative, String displayName,
       GenericUDF genericUDF) {
     this.isNative = isNative;
     this.displayName = displayName;
     this.genericUDF = genericUDF;
+    this.isInternalTableFunction = false;
   }
 
   public FunctionInfo(boolean isNative, String displayName,
@@ -52,6 +59,7 @@ public class FunctionInfo {
     this.isNative = isNative;
     this.displayName = displayName;
     this.genericUDAFResolver = genericUDAFResolver;
+    this.isInternalTableFunction = false;
   }
 
   public FunctionInfo(boolean isNative, String displayName,
@@ -59,6 +67,16 @@ public class FunctionInfo {
     this.isNative = isNative;
     this.displayName = displayName;
     this.genericUDTF = genericUDTF;
+    this.isInternalTableFunction = false;
+  }
+
+  public FunctionInfo(String displayName, Class<? extends TableFunctionResolver> tFnCls)
+  {
+    this.displayName = displayName;
+    this.tableFunctionResolver = tFnCls;
+    PartitionTableFunctionDescription def = tableFunctionResolver.getAnnotation(PartitionTableFunctionDescription.class);
+    this.isNative = (def == null) ? false : def.isInternal();
+    this.isInternalTableFunction = isNative;
   }
 
   /**
@@ -90,6 +108,8 @@ public class FunctionInfo {
     return genericUDAFResolver;
   }
 
+
+
   /**
    * Get the Class of the UDF.
    */
@@ -109,6 +129,9 @@ public class FunctionInfo {
     } else if (isGenericUDTF()) {
       return genericUDTF.getClass();
     }
+    if(isTableFunction()) {
+      return this.tableFunctionResolver;
+    }
     return null;
   }
 
@@ -131,6 +154,14 @@ public class FunctionInfo {
   }
 
   /**
+   * Internal table functions cannot be used in the language.
+   * {@link WindowingTableFunction}
+   */
+  public boolean isInternalTableFunction() {
+    return isInternalTableFunction;
+  }
+
+  /**
    * @return TRUE if the function is a GenericUDF
    */
   public boolean isGenericUDF() {
@@ -150,4 +181,11 @@ public class FunctionInfo {
   public boolean isGenericUDTF() {
     return null != genericUDTF;
   }
+
+  /**
+   * @return TRUE if the function is a Table Function
+   */
+  public boolean isTableFunction() {
+    return null != tableFunctionResolver;
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1476646&r1=1476645&r2=1476646&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Sat Apr 27
17:53:48 2013
@@ -182,7 +182,6 @@ public final class FunctionRegistry {
   public static final String NOOP_TABLE_FUNCTION = "noop";
   public static final String NOOP_MAP_TABLE_FUNCTION = "noopwithmap";
 
-  static Map<String, PTFFunctionInfo> tableFunctions = Collections.synchronizedMap(new
LinkedHashMap<String, PTFFunctionInfo>());
   static Map<String, WindowFunctionInfo> windowFunctions = Collections.synchronizedMap(new
LinkedHashMap<String, WindowFunctionInfo>());
 
   /*
@@ -1294,6 +1293,9 @@ public final class FunctionRegistry {
       FunctionRegistry.registerTemporaryGenericUDAF(
         functionName, (GenericUDAFResolver)
         ReflectionUtils.newInstance(udfClass, null));
+    } else if(TableFunctionResolver.class.isAssignableFrom(udfClass)) {
+      FunctionRegistry.registerTableFunction(
+        functionName, (Class<? extends TableFunctionResolver>)udfClass);
     } else {
       return false;
     }
@@ -1406,14 +1408,17 @@ public final class FunctionRegistry {
 
   public static boolean isTableFunction(String name)
   {
-    PTFFunctionInfo tFInfo = tableFunctions.get(name.toLowerCase());
-     return tFInfo != null && !tFInfo.isInternal();
+    FunctionInfo tFInfo = mFunctions.get(name.toLowerCase());
+    return tFInfo != null && !tFInfo.isInternalTableFunction() && tFInfo.isTableFunction();
   }
 
   public static TableFunctionResolver getTableFunctionResolver(String name)
   {
-    PTFFunctionInfo tfInfo = tableFunctions.get(name.toLowerCase());
-    return (TableFunctionResolver) ReflectionUtils.newInstance(tfInfo.getFunctionResolver(),
null);
+    FunctionInfo tfInfo = mFunctions.get(name.toLowerCase());
+    if(tfInfo.isTableFunction()) {
+      return (TableFunctionResolver) ReflectionUtils.newInstance(tfInfo.getFunctionClass(),
null);
+    }
+    return null;
   }
 
   public static TableFunctionResolver getWindowingTableFunction()
@@ -1428,8 +1433,8 @@ public final class FunctionRegistry {
 
   public static void registerTableFunction(String name, Class<? extends TableFunctionResolver>
tFnCls)
   {
-    PTFFunctionInfo tInfo = new PTFFunctionInfo(name, tFnCls);
-    tableFunctions.put(name.toLowerCase(), tInfo);
+    FunctionInfo tInfo = new FunctionInfo(name, tFnCls);
+    mFunctions.put(name.toLowerCase(), tInfo);
   }
 
 }

Added: hive/trunk/ql/src/test/queries/clientpositive/ptf_register_tblfn.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/ptf_register_tblfn.q?rev=1476646&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/ptf_register_tblfn.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/ptf_register_tblfn.q Sat Apr 27 17:53:48
2013
@@ -0,0 +1,29 @@
+DROP TABLE flights_tiny;
+
+create table flights_tiny ( 
+ORIGIN_CITY_NAME string, 
+DEST_CITY_NAME string, 
+YEAR int, 
+MONTH int, 
+DAY_OF_MONTH int, 
+ARR_DELAY float, 
+FL_NUM string 
+);
+
+LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny;
+
+create temporary function npathtest as 'org.apache.hadoop.hive.ql.udf.ptf.NPath$NPathResolver';
+
+
+-- 1. basic Npath test
+select origin_city_name, fl_num, year, month, day_of_month, sz, tpath 
+from npathtest(on 
+        flights_tiny 
+        distribute by fl_num 
+        sort by year, month, day_of_month  
+      arg1('LATE.LATE+'), 
+      arg2('LATE'), arg3(arr_delay > 15), 
+    arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month
as tpath') 
+   );
+
+drop temporary function npathtest;

Added: hive/trunk/ql/src/test/results/clientpositive/ptf_register_tblfn.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ptf_register_tblfn.q.out?rev=1476646&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ptf_register_tblfn.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/ptf_register_tblfn.q.out Sat Apr 27 17:53:48
2013
@@ -0,0 +1,81 @@
+PREHOOK: query: DROP TABLE flights_tiny
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE flights_tiny
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table flights_tiny ( 
+ORIGIN_CITY_NAME string, 
+DEST_CITY_NAME string, 
+YEAR int, 
+MONTH int, 
+DAY_OF_MONTH int, 
+ARR_DELAY float, 
+FL_NUM string 
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table flights_tiny ( 
+ORIGIN_CITY_NAME string, 
+DEST_CITY_NAME string, 
+YEAR int, 
+MONTH int, 
+DAY_OF_MONTH int, 
+ARR_DELAY float, 
+FL_NUM string 
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@flights_tiny
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt' OVERWRITE INTO TABLE
flights_tiny
+PREHOOK: type: LOAD
+PREHOOK: Output: default@flights_tiny
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt' OVERWRITE INTO TABLE
flights_tiny
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@flights_tiny
+PREHOOK: query: create temporary function npathtest as 'org.apache.hadoop.hive.ql.udf.ptf.NPath$NPathResolver'
+PREHOOK: type: CREATEFUNCTION
+POSTHOOK: query: create temporary function npathtest as 'org.apache.hadoop.hive.ql.udf.ptf.NPath$NPathResolver'
+POSTHOOK: type: CREATEFUNCTION
+PREHOOK: query: -- 1. basic Npath test
+select origin_city_name, fl_num, year, month, day_of_month, sz, tpath 
+from npathtest(on 
+        flights_tiny 
+        distribute by fl_num 
+        sort by year, month, day_of_month  
+      arg1('LATE.LATE+'), 
+      arg2('LATE'), arg3(arr_delay > 15), 
+    arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month
as tpath') 
+   )
+PREHOOK: type: QUERY
+PREHOOK: Input: default@flights_tiny
+#### A masked pattern was here ####
+POSTHOOK: query: -- 1. basic Npath test
+select origin_city_name, fl_num, year, month, day_of_month, sz, tpath 
+from npathtest(on 
+        flights_tiny 
+        distribute by fl_num 
+        sort by year, month, day_of_month  
+      arg1('LATE.LATE+'), 
+      arg2('LATE'), arg3(arr_delay > 15), 
+    arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month
as tpath') 
+   )
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@flights_tiny
+#### A masked pattern was here ####
+Baltimore	1142	2010	10	20	6	20
+Baltimore	1142	2010	10	21	5	21
+Baltimore	1142	2010	10	22	4	22
+Baltimore	1142	2010	10	25	3	25
+Baltimore	1142	2010	10	26	2	26
+Chicago	1531	2010	10	21	2	21
+Chicago	1531	2010	10	25	3	25
+Chicago	1531	2010	10	26	2	26
+Baltimore	1599	2010	10	21	2	21
+Baltimore	1599	2010	10	25	3	25
+Baltimore	1599	2010	10	26	2	26
+Chicago	361	2010	10	20	2	20
+Washington	7291	2010	10	27	2	27
+Chicago	897	2010	10	20	4	20
+Chicago	897	2010	10	21	3	21
+Chicago	897	2010	10	22	2	22
+PREHOOK: query: drop temporary function npathtest
+PREHOOK: type: DROPFUNCTION
+POSTHOOK: query: drop temporary function npathtest
+POSTHOOK: type: DROPFUNCTION

Modified: hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out?rev=1476646&r1=1476645&r2=1476646&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out Sat Apr 27 17:53:48
2013
@@ -110,7 +110,10 @@ month
 named_struct
 negative
 ngrams
+noop
+noopwithmap
 not
+npath
 ntile
 nvl
 or
@@ -174,6 +177,7 @@ var_samp
 variance
 weekofyear
 when
+windowingtablefunction
 xpath
 xpath_boolean
 xpath_double



Mime
View raw message