hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c..@apache.org
Subject svn commit: r1234150 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/udf/generic/ test/queries/clientnegative/ test/queries/clientpositive/ test/results/clientnegative/ test/results/clientpositive/
Date Fri, 20 Jan 2012 21:32:25 GMT
Author: cws
Date: Fri Jan 20 21:32:24 2012
New Revision: 1234150

URL: http://svn.apache.org/viewvc?rev=1234150&view=rev
Log:
HIVE-2203. Extend concat_ws() UDF to support arrays of strings (Zhenxiao Luo via cws)

Added:
    hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong1.q
    hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong2.q
    hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong3.q
    hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong1.q.out
    hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong2.q.out
    hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong3.q.out
Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcatWS.java
    hive/trunk/ql/src/test/queries/clientpositive/udf_concat_ws.q
    hive/trunk/ql/src/test/results/clientpositive/udf_concat_ws.q.out

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcatWS.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcatWS.java?rev=1234150&r1=1234149&r2=1234150&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcatWS.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcatWS.java Fri
Jan 20 21:32:24 2012
@@ -24,24 +24,28 @@ import org.apache.hadoop.hive.ql.exec.UD
 import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
 import org.apache.hadoop.io.Text;
 
 /**
  * Generic UDF for string function
- * <code>CONCAT_WS(sep,str1,str2,str3,...)</code>. This mimics the function from
+ * <code>CONCAT_WS(sep, [string | array(string)]+)<code>.
+ * This mimics the function from
  * MySQL http://dev.mysql.com/doc/refman/5.0/en/string-functions.html#
  * function_concat-ws
- * 
+ *
  * @see org.apache.hadoop.hive.ql.udf.generic.GenericUDF
  */
-@Description(name = "concat_ws", value = "_FUNC_(separator, str1, str2, ...) - "
+@Description(name = "concat_ws",
+    value = "_FUNC_(separator, [string | array(string)]+) - "
     + "returns the concatenation of the strings separated by the separator.",
     extended = "Example:\n"
-    + "  > SELECT _FUNC_('ce', 'fa', 'book') FROM src LIMIT 1;\n"
-    + "  'facebook'")
+    + "  > SELECT _FUNC_('.', 'www', array('facebook', 'com')) FROM src LIMIT 1;\n"
+    + "  'www.facebook.com'")
 public class GenericUDFConcatWS extends GenericUDF {
   private ObjectInspector[] argumentOIs;
 
@@ -49,15 +53,28 @@ public class GenericUDFConcatWS extends 
   public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException
{
     if (arguments.length < 2) {
       throw new UDFArgumentLengthException(
-          "The function CONCAT_WS(separator,str1,str2,str3,...) needs at least two arguments.");
+          "The function CONCAT_WS(separator,[string | array(string)]+) "
+            + "needs at least two arguments.");
     }
 
+    // check if argument is a string or an array of strings
     for (int i = 0; i < arguments.length; i++) {
-      if (arguments[i].getTypeName() != Constants.STRING_TYPE_NAME
-          && arguments[i].getTypeName() != Constants.VOID_TYPE_NAME) {
-        throw new UDFArgumentTypeException(i, "Argument " + (i + 1)
+      switch(arguments[i].getCategory()) {
+        case LIST:
+          if (((ListObjectInspector)arguments[i]).getListElementObjectInspector()
+            .getTypeName().equals(Constants.STRING_TYPE_NAME)
+            || ((ListObjectInspector)arguments[i]).getListElementObjectInspector()
+            .getTypeName().equals(Constants.VOID_TYPE_NAME))
+          break;
+        case PRIMITIVE:
+          if (arguments[i].getTypeName().equals(Constants.STRING_TYPE_NAME)
+            || arguments[i].getTypeName().equals(Constants.VOID_TYPE_NAME))
+          break;
+        default:
+          throw new UDFArgumentTypeException(i, "Argument " + (i + 1)
             + " of function CONCAT_WS must be \"" + Constants.STRING_TYPE_NAME
-            + "\", but \"" + arguments[i].getTypeName() + "\" was found.");
+            + " or " + Constants.LIST_TYPE_NAME + "<" + Constants.STRING_TYPE_NAME
+            + ">\", but \"" + arguments[i].getTypeName() + "\" was found.");
       }
     }
 
@@ -84,8 +101,22 @@ public class GenericUDFConcatWS extends 
         } else {
           sb.append(separator);
         }
-        sb.append(((StringObjectInspector) argumentOIs[i])
+        if (argumentOIs[i].getCategory().equals(Category.LIST)) {
+          Object strArray = arguments[i].get();
+          ListObjectInspector strArrayOI = (ListObjectInspector) argumentOIs[i];
+          boolean strArrayFirst = true;
+          for (int j = 0; j < strArrayOI.getListLength(strArray); j++) {
+            if (strArrayFirst) {
+              strArrayFirst = false;
+            } else {
+              sb.append(separator);
+            }
+            sb.append(strArrayOI.getListElement(strArray, j));
+          }
+        } else {
+          sb.append(((StringObjectInspector) argumentOIs[i])
             .getPrimitiveJavaObject(arguments[i].get()));
+        }
       }
     }
 

Added: hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong1.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong1.q?rev=1234150&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong1.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong1.q Fri Jan 20 21:32:24
2012
@@ -0,0 +1,2 @@
+-- invalid argument number
+SELECT concat_ws('-') FROM src LIMIT 1;

Added: hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong2.q?rev=1234150&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong2.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong2.q Fri Jan 20 21:32:24
2012
@@ -0,0 +1,2 @@
+-- invalid argument type
+SELECT concat_ws('[]', array(100, 200, 50)) FROM src LIMIT 1;

Added: hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong3.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong3.q?rev=1234150&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong3.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/udf_concat_ws_wrong3.q Fri Jan 20 21:32:24
2012
@@ -0,0 +1,2 @@
+-- invalid argument type
+SELECT concat_ws(1234, array('www', 'facebook', 'com')) FROM src LIMIT 1;

Modified: hive/trunk/ql/src/test/queries/clientpositive/udf_concat_ws.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/udf_concat_ws.q?rev=1234150&r1=1234149&r2=1234150&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/udf_concat_ws.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/udf_concat_ws.q Fri Jan 20 21:32:24 2012
@@ -16,3 +16,28 @@ SELECT concat_ws(dest1.c1, dest1.c2, des
        concat_ws(NULL, dest1.c1, dest1.c2, dest1.c3),
        concat_ws('**', dest1.c1, NULL, dest1.c3) FROM dest1;
 
+-- evalutes function for array of strings
+EXPLAIN
+SELECT concat_ws('.', array('www', 'face', 'book', 'com'), '1234'),
+       concat_ws('-', 'www', array('face', 'book', 'com'), '1234'),
+       concat_ws('F', 'www', array('face', 'book', 'com', '1234')),
+       concat_ws('_', array('www', 'face'), array('book', 'com', '1234')),
+       concat_ws('**', 'www', array('face'), array('book', 'com', '1234')),
+       concat_ws('[]', array('www'), 'face', array('book', 'com', '1234')),
+       concat_ws('AAA', array('www'), array('face', 'book', 'com'), '1234') FROM dest1 LIMIT
1;
+
+SELECT concat_ws('.', array('www', 'face', 'book', 'com'), '1234'),
+       concat_ws('-', 'www', array('face', 'book', 'com'), '1234'),
+       concat_ws('F', 'www', array('face', 'book', 'com', '1234')),
+       concat_ws('_', array('www', 'face'), array('book', 'com', '1234')),
+       concat_ws('**', 'www', array('face'), array('book', 'com', '1234')),
+       concat_ws('[]', array('www'), 'face', array('book', 'com', '1234')),
+       concat_ws('AAA', array('www'), array('face', 'book', 'com'), '1234') FROM dest1 LIMIT
1;
+
+SELECT concat_ws(NULL, array('www', 'face', 'book', 'com'), '1234'),
+       concat_ws(NULL, 'www', array('face', 'book', 'com'), '1234'),
+       concat_ws(NULL, 'www', array('face', 'book', 'com', '1234')),
+       concat_ws(NULL, array('www', 'face'), array('book', 'com', '1234')),
+       concat_ws(NULL, 'www', array('face'), array('book', 'com', '1234')),
+       concat_ws(NULL, array('www'), 'face', array('book', 'com', '1234')),
+       concat_ws(NULL, array('www'), array('face', 'book', 'com'), '1234') FROM dest1 LIMIT
1;

Added: hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong1.q.out?rev=1234150&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong1.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong1.q.out Fri Jan 20 21:32:24
2012
@@ -0,0 +1 @@
+FAILED: Error in semantic analysis: Line 2:7 Arguments length mismatch ''-'': The function
CONCAT_WS(separator,[string | array(string)]+) needs at least two arguments.

Added: hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong2.q.out?rev=1234150&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong2.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong2.q.out Fri Jan 20 21:32:24
2012
@@ -0,0 +1 @@
+FAILED: Error in semantic analysis: Line 2:23 Argument type mismatch '50': Argument 2 of
function CONCAT_WS must be "string or array<string>", but "array<int>" was found.

Added: hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong3.q.out?rev=1234150&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong3.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/udf_concat_ws_wrong3.q.out Fri Jan 20 21:32:24
2012
@@ -0,0 +1 @@
+FAILED: Error in semantic analysis: Line 2:17 Argument type mismatch '1234': Argument 1 of
function CONCAT_WS must be "string or array<string>", but "int" was found.

Modified: hive/trunk/ql/src/test/results/clientpositive/udf_concat_ws.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udf_concat_ws.q.out?rev=1234150&r1=1234149&r2=1234150&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udf_concat_ws.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/udf_concat_ws.q.out Fri Jan 20 21:32:24
2012
@@ -2,15 +2,15 @@ PREHOOK: query: DESCRIBE FUNCTION concat
 PREHOOK: type: DESCFUNCTION
 POSTHOOK: query: DESCRIBE FUNCTION concat_ws
 POSTHOOK: type: DESCFUNCTION
-concat_ws(separator, str1, str2, ...) - returns the concatenation of the strings separated
by the separator.
+concat_ws(separator, [string | array(string)]+) - returns the concatenation of the strings
separated by the separator.
 PREHOOK: query: DESCRIBE FUNCTION EXTENDED concat_ws
 PREHOOK: type: DESCFUNCTION
 POSTHOOK: query: DESCRIBE FUNCTION EXTENDED concat_ws
 POSTHOOK: type: DESCFUNCTION
-concat_ws(separator, str1, str2, ...) - returns the concatenation of the strings separated
by the separator.
+concat_ws(separator, [string | array(string)]+) - returns the concatenation of the strings
separated by the separator.
 Example:
-  > SELECT concat_ws('ce', 'fa', 'book') FROM src LIMIT 1;
-  'facebook'
+  > SELECT concat_ws('.', 'www', array('facebook', 'com')) FROM src LIMIT 1;
+  'www.facebook.com'
 PREHOOK: query: CREATE TABLE dest1(c1 STRING, c2 STRING, c3 STRING)
 PREHOOK: type: CREATETABLE
 POSTHOOK: query: CREATE TABLE dest1(c1 STRING, c2 STRING, c3 STRING)
@@ -97,3 +97,118 @@ POSTHOOK: Lineage: dest1.c1 SIMPLE []
 POSTHOOK: Lineage: dest1.c2 SIMPLE []
 POSTHOOK: Lineage: dest1.c3 SIMPLE []
 xyzabc8675309	abc,xyz,8675309	NULL	abc**8675309
+PREHOOK: query: -- evalutes function for array of strings
+EXPLAIN
+SELECT concat_ws('.', array('www', 'face', 'book', 'com'), '1234'),
+       concat_ws('-', 'www', array('face', 'book', 'com'), '1234'),
+       concat_ws('F', 'www', array('face', 'book', 'com', '1234')),
+       concat_ws('_', array('www', 'face'), array('book', 'com', '1234')),
+       concat_ws('**', 'www', array('face'), array('book', 'com', '1234')),
+       concat_ws('[]', array('www'), 'face', array('book', 'com', '1234')),
+       concat_ws('AAA', array('www'), array('face', 'book', 'com'), '1234') FROM dest1 LIMIT
1
+PREHOOK: type: QUERY
+POSTHOOK: query: -- evalutes function for array of strings
+EXPLAIN
+SELECT concat_ws('.', array('www', 'face', 'book', 'com'), '1234'),
+       concat_ws('-', 'www', array('face', 'book', 'com'), '1234'),
+       concat_ws('F', 'www', array('face', 'book', 'com', '1234')),
+       concat_ws('_', array('www', 'face'), array('book', 'com', '1234')),
+       concat_ws('**', 'www', array('face'), array('book', 'com', '1234')),
+       concat_ws('[]', array('www'), 'face', array('book', 'com', '1234')),
+       concat_ws('AAA', array('www'), array('face', 'book', 'com'), '1234') FROM dest1 LIMIT
1
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: dest1.c1 SIMPLE []
+POSTHOOK: Lineage: dest1.c2 SIMPLE []
+POSTHOOK: Lineage: dest1.c3 SIMPLE []
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME dest1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR
TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION concat_ws '.' (TOK_FUNCTION array 'www'
'face' 'book' 'com') '1234')) (TOK_SELEXPR (TOK_FUNCTION concat_ws '-' 'www' (TOK_FUNCTION
array 'face' 'book' 'com') '1234')) (TOK_SELEXPR (TOK_FUNCTION concat_ws 'F' 'www' (TOK_FUNCTION
array 'face' 'book' 'com' '1234'))) (TOK_SELEXPR (TOK_FUNCTION concat_ws '_' (TOK_FUNCTION
array 'www' 'face') (TOK_FUNCTION array 'book' 'com' '1234'))) (TOK_SELEXPR (TOK_FUNCTION
concat_ws '**' 'www' (TOK_FUNCTION array 'face') (TOK_FUNCTION array 'book' 'com' '1234')))
(TOK_SELEXPR (TOK_FUNCTION concat_ws '[]' (TOK_FUNCTION array 'www') 'face' (TOK_FUNCTION
array 'book' 'com' '1234'))) (TOK_SELEXPR (TOK_FUNCTION concat_ws 'AAA' (TOK_FUNCTION array
'www') (TOK_FUNCTION array 'face' 'book' 'com') '1234'))) (TOK_LIMIT 1)))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        dest1 
+          TableScan
+            alias: dest1
+            Select Operator
+              expressions:
+                    expr: concat_ws('.', array('www','face','book','com'), '1234')
+                    type: string
+                    expr: concat_ws('-', 'www', array('face','book','com'), '1234')
+                    type: string
+                    expr: concat_ws('F', 'www', array('face','book','com','1234'))
+                    type: string
+                    expr: concat_ws('_', array('www','face'), array('book','com','1234'))
+                    type: string
+                    expr: concat_ws('**', 'www', array('face'), array('book','com','1234'))
+                    type: string
+                    expr: concat_ws('[]', array('www'), 'face', array('book','com','1234'))
+                    type: string
+                    expr: concat_ws('AAA', array('www'), array('face','book','com'), '1234')
+                    type: string
+              outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+              Limit
+                File Output Operator
+                  compressed: false
+                  GlobalTableId: 0
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: 1
+
+
+PREHOOK: query: SELECT concat_ws('.', array('www', 'face', 'book', 'com'), '1234'),
+       concat_ws('-', 'www', array('face', 'book', 'com'), '1234'),
+       concat_ws('F', 'www', array('face', 'book', 'com', '1234')),
+       concat_ws('_', array('www', 'face'), array('book', 'com', '1234')),
+       concat_ws('**', 'www', array('face'), array('book', 'com', '1234')),
+       concat_ws('[]', array('www'), 'face', array('book', 'com', '1234')),
+       concat_ws('AAA', array('www'), array('face', 'book', 'com'), '1234') FROM dest1 LIMIT
1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT concat_ws('.', array('www', 'face', 'book', 'com'), '1234'),
+       concat_ws('-', 'www', array('face', 'book', 'com'), '1234'),
+       concat_ws('F', 'www', array('face', 'book', 'com', '1234')),
+       concat_ws('_', array('www', 'face'), array('book', 'com', '1234')),
+       concat_ws('**', 'www', array('face'), array('book', 'com', '1234')),
+       concat_ws('[]', array('www'), 'face', array('book', 'com', '1234')),
+       concat_ws('AAA', array('www'), array('face', 'book', 'com'), '1234') FROM dest1 LIMIT
1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+#### A masked pattern was here ####
+POSTHOOK: Lineage: dest1.c1 SIMPLE []
+POSTHOOK: Lineage: dest1.c2 SIMPLE []
+POSTHOOK: Lineage: dest1.c3 SIMPLE []
+www.face.book.com.1234	www-face-book-com-1234	wwwFfaceFbookFcomF1234	www_face_book_com_1234
www**face**book**com**1234	www[]face[]book[]com[]1234	wwwAAAfaceAAAbookAAAcomAAA1234
+PREHOOK: query: SELECT concat_ws(NULL, array('www', 'face', 'book', 'com'), '1234'),
+       concat_ws(NULL, 'www', array('face', 'book', 'com'), '1234'),
+       concat_ws(NULL, 'www', array('face', 'book', 'com', '1234')),
+       concat_ws(NULL, array('www', 'face'), array('book', 'com', '1234')),
+       concat_ws(NULL, 'www', array('face'), array('book', 'com', '1234')),
+       concat_ws(NULL, array('www'), 'face', array('book', 'com', '1234')),
+       concat_ws(NULL, array('www'), array('face', 'book', 'com'), '1234') FROM dest1 LIMIT
1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT concat_ws(NULL, array('www', 'face', 'book', 'com'), '1234'),
+       concat_ws(NULL, 'www', array('face', 'book', 'com'), '1234'),
+       concat_ws(NULL, 'www', array('face', 'book', 'com', '1234')),
+       concat_ws(NULL, array('www', 'face'), array('book', 'com', '1234')),
+       concat_ws(NULL, 'www', array('face'), array('book', 'com', '1234')),
+       concat_ws(NULL, array('www'), 'face', array('book', 'com', '1234')),
+       concat_ws(NULL, array('www'), array('face', 'book', 'com'), '1234') FROM dest1 LIMIT
1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+#### A masked pattern was here ####
+POSTHOOK: Lineage: dest1.c1 SIMPLE []
+POSTHOOK: Lineage: dest1.c2 SIMPLE []
+POSTHOOK: Lineage: dest1.c3 SIMPLE []
+NULL	NULL	NULL	NULL	NULL	NULL	NULL



Mime
View raw message