hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sd...@apache.org
Subject svn commit: r1341589 - in /hive/trunk: bin/ common/src/java/org/apache/hadoop/hive/conf/ conf/ data/scripts/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/metadata/ ql/src/test/queries/clientpositive/ ql/src/test/res...
Date Tue, 22 May 2012 18:37:38 GMT
Author: sdong
Date: Tue May 22 18:37:37 2012
New Revision: 1341589

URL: http://svn.apache.org/viewvc?rev=1341589&view=rev
Log:
HIVE-3030 escape more chars for script operator (Namit Jain via Siying Dong)

Added:
    hive/trunk/data/scripts/doubleescapedtab.py
    hive/trunk/data/scripts/escapedcarriagereturn.py
    hive/trunk/data/scripts/escapednewline.py
    hive/trunk/data/scripts/escapedtab.py
Modified:
    hive/trunk/bin/hive
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/conf/hive-default.xml.template
    hive/trunk/data/scripts/newline.py
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordWriter.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java
    hive/trunk/ql/src/test/queries/clientpositive/newline.q
    hive/trunk/ql/src/test/results/clientpositive/newline.q.out

Modified: hive/trunk/bin/hive
URL: http://svn.apache.org/viewvc/hive/trunk/bin/hive?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/bin/hive (original)
+++ hive/trunk/bin/hive Tue May 22 18:37:37 2012
@@ -216,6 +216,7 @@ if [ "$DEBUG" ]; then
   else
     get_debug_params "$DEBUG"
     export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS $HIVE_MAIN_CLIENT_DEBUG_OPTS"
+    export HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
   fi
 fi
 

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue May 22 18:37:37
2012
@@ -389,7 +389,7 @@ public class HiveConf extends Configurat
         "org.apache.hadoop.hive.ql.exec.TextRecordReader"),
     HIVESCRIPTRECORDWRITER("hive.script.recordwriter",
         "org.apache.hadoop.hive.ql.exec.TextRecordWriter"),
-    HIVESCRIPTESCAPENEWLINES("hive.script.escape.newlines", false),
+    HIVESCRIPTESCAPE("hive.transform.escape.input", false),
 
     // HWI
     HIVEHWILISTENHOST("hive.hwi.listen.host", "0.0.0.0"),

Modified: hive/trunk/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Tue May 22 18:37:37 2012
@@ -1273,12 +1273,12 @@
 </property>
 
 <property>
-  <name>hive.script.escape.newlines</name>
+  <name>hive.transform.escape.input</name>
   <value>false</value>
   <description>
-    This adds an option to escape the newlines when they are passed to the 
-    user script. This is useful is the hive tables can contain data that
-    can contain newlines.
+    This adds an option to escape special chars (newlines, carriage returns and
+    tabs) when they are passed to the user script. This is useful if the hive tables
+    can contain data that contains special characters.
   </description>
 </property>
 

Added: hive/trunk/data/scripts/doubleescapedtab.py
URL: http://svn.apache.org/viewvc/hive/trunk/data/scripts/doubleescapedtab.py?rev=1341589&view=auto
==============================================================================
--- hive/trunk/data/scripts/doubleescapedtab.py (added)
+++ hive/trunk/data/scripts/doubleescapedtab.py Tue May 22 18:37:37 2012
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import sys
+
+for line in sys.stdin:
+  print "1\\\\\\t2"
+  print "1\\\\\\\\t2"
+

Added: hive/trunk/data/scripts/escapedcarriagereturn.py
URL: http://svn.apache.org/viewvc/hive/trunk/data/scripts/escapedcarriagereturn.py?rev=1341589&view=auto
==============================================================================
--- hive/trunk/data/scripts/escapedcarriagereturn.py (added)
+++ hive/trunk/data/scripts/escapedcarriagereturn.py Tue May 22 18:37:37 2012
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import sys
+
+for line in sys.stdin:
+  print "1\\\\r2"
+

Added: hive/trunk/data/scripts/escapednewline.py
URL: http://svn.apache.org/viewvc/hive/trunk/data/scripts/escapednewline.py?rev=1341589&view=auto
==============================================================================
--- hive/trunk/data/scripts/escapednewline.py (added)
+++ hive/trunk/data/scripts/escapednewline.py Tue May 22 18:37:37 2012
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import sys
+
+for line in sys.stdin:
+  print "1\\\\n2"
+

Added: hive/trunk/data/scripts/escapedtab.py
URL: http://svn.apache.org/viewvc/hive/trunk/data/scripts/escapedtab.py?rev=1341589&view=auto
==============================================================================
--- hive/trunk/data/scripts/escapedtab.py (added)
+++ hive/trunk/data/scripts/escapedtab.py Tue May 22 18:37:37 2012
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import sys
+
+for line in sys.stdin:
+  print "1\\\\t2"
+

Modified: hive/trunk/data/scripts/newline.py
URL: http://svn.apache.org/viewvc/hive/trunk/data/scripts/newline.py?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/data/scripts/newline.py (original)
+++ hive/trunk/data/scripts/newline.py Tue May 22 18:37:37 2012
@@ -20,3 +20,5 @@ import sys
 
 for line in sys.stdin:
   print "1\\n2"
+  print "1\\r2"
+  print "1\\t2"

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java Tue May 22
18:37:37 2012
@@ -60,8 +60,8 @@ public class TextRecordReader implements
 
     int bytesConsumed = lineReader.readLine((Text) row);
 
-    if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESCRIPTESCAPENEWLINES)) {
-      return HiveUtils.unescapeNewLine((Text) row);
+    if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESCRIPTESCAPE)) {
+      return HiveUtils.unescapeText((Text) row);
     }
     return bytesConsumed;
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordWriter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordWriter.java?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordWriter.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordWriter.java Tue May 22
18:37:37 2012
@@ -46,8 +46,8 @@ public class TextRecordWriter implements
     Text text = (Text) row;
     Text escapeText = text;
 
-    if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESCRIPTESCAPENEWLINES)) {
-      escapeText = HiveUtils.escapeNewLine(text);
+    if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESCRIPTESCAPE)) {
+      escapeText = HiveUtils.escapeText(text);
     }
 
     out.write(escapeText.getBytes(), 0, escapeText.getLength());

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java Tue May 22 18:37:37
2012
@@ -97,10 +97,16 @@ public final class HiveUtils {
     return (escape.toString());
   }
 
+  static final byte[] escapeEscapeBytes = "\\\\".getBytes();;
+  static final byte[] escapeUnescapeBytes = "\\".getBytes();
   static final byte[] newLineEscapeBytes = "\\n".getBytes();;
   static final byte[] newLineUnescapeBytes = "\n".getBytes();
+  static final byte[] carriageReturnEscapeBytes = "\\r".getBytes();;
+  static final byte[] carriageReturnUnescapeBytes = "\r".getBytes();
+  static final byte[] tabEscapeBytes = "\\t".getBytes();;
+  static final byte[] tabUnescapeBytes = "\t".getBytes();
 
-  public static Text escapeNewLine(Text text) {
+  public static Text escapeText(Text text) {
     int length = text.getLength();
     byte[] textBytes = text.getBytes();
 
@@ -109,20 +115,50 @@ public final class HiveUtils {
 
     for (int i = 0; i < length; ++i) {
       int c = text.charAt(i);
+      byte[] escaped;
+      int start;
+      int len;
+
       switch (c) {
+
+      case '\\':
+        escaped = escapeEscapeBytes;
+        start = 0;
+        len = escaped.length;
+        break;
+
       case '\n':
-        byte[] escaped = newLineEscapeBytes;
-        escape.append(escaped, 0, escaped.length);
+        escaped = newLineEscapeBytes;
+        start = 0;
+        len = escaped.length;
         break;
+
+      case '\r':
+        escaped = carriageReturnEscapeBytes;
+        start = 0;
+        len = escaped.length;
+        break;
+
+      case '\t':
+        escaped = tabEscapeBytes;
+        start = 0;
+        len = escaped.length;
+        break;
+
       default:
-        escape.append(textBytes, i, 1);
+        escaped = textBytes;
+        start = i;
+        len = 1;
         break;
       }
+
+      escape.append(escaped, start, len);
+
     }
     return escape;
   }
 
-  public static int unescapeNewLine(Text text) {
+  public static int unescapeText(Text text) {
     Text escape = new Text(text);
     text.clear();
 
@@ -136,8 +172,11 @@ public final class HiveUtils {
       case '\\':
         if (hadSlash) {
           text.append(textBytes, i, 1);
+          hadSlash = false;
+        }
+        else {
+          hadSlash = true;
         }
-        hadSlash = true;
         break;
       case 'n':
         if (hadSlash) {
@@ -149,6 +188,28 @@ public final class HiveUtils {
         }
         hadSlash = false;
         break;
+      case 'r':
+        if (hadSlash) {
+          byte[] carriageReturn = carriageReturnUnescapeBytes;
+          text.append(carriageReturn, 0, carriageReturn.length);
+        }
+        else {
+          text.append(textBytes, i, 1);
+        }
+        hadSlash = false;
+        break;
+
+      case 't':
+        if (hadSlash) {
+          byte[] tab = tabUnescapeBytes;
+          text.append(tab, 0, tab.length);
+        }
+        else {
+          text.append(textBytes, i, 1);
+        }
+        hadSlash = false;
+        break;
+
       default:
         if (hadSlash) {
           text.append(textBytes, i-1, 1);

Modified: hive/trunk/ql/src/test/queries/clientpositive/newline.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/newline.q?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/newline.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/newline.q Tue May 22 18:37:37 2012
@@ -1,9 +1,53 @@
 add file ../data/scripts/newline.py;
-set hive.script.escape.newlines=true;
+set hive.transform.escape.input=true;
 
 create table tmp_tmp(key string, value string) stored as rcfile;
 insert overwrite table tmp_tmp
 SELECT TRANSFORM(key, value) USING
-'python newline.py' AS key, value FROM src limit 5;
+'python newline.py' AS key, value FROM src limit 6;
 
-select * from tmp_tmp;
\ No newline at end of file
+select * from tmp_tmp;
+
+drop table tmp_tmp;
+
+add file ../data/scripts/escapednewline.py;
+add file ../data/scripts/escapedtab.py;
+add file ../data/scripts/doubleescapedtab.py;
+add file ../data/scripts/escapedcarriagereturn.py;
+
+create table tmp_tmp(key string, value string) stored as rcfile;
+insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapednewline.py' AS key, value FROM src limit 5;
+
+select * from tmp_tmp;
+
+SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp;
+
+insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapedcarriagereturn.py' AS key, value FROM src limit 5;
+
+select * from tmp_tmp;
+
+SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp;
+
+insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapedtab.py' AS key, value FROM src limit 5;
+
+select * from tmp_tmp;
+
+SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp;
+
+insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python doubleescapedtab.py' AS key, value FROM src limit 5;
+
+select * from tmp_tmp;
+
+SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp;

Modified: hive/trunk/ql/src/test/results/clientpositive/newline.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/newline.q.out?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/newline.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/newline.q.out Tue May 22 18:37:37 2012
@@ -5,13 +5,13 @@ POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: default@tmp_tmp
 PREHOOK: query: insert overwrite table tmp_tmp
 SELECT TRANSFORM(key, value) USING
-'python newline.py' AS key, value FROM src limit 5
+'python newline.py' AS key, value FROM src limit 6
 PREHOOK: type: QUERY
 PREHOOK: Input: default@src
 PREHOOK: Output: default@tmp_tmp
 POSTHOOK: query: insert overwrite table tmp_tmp
 SELECT TRANSFORM(key, value) USING
-'python newline.py' AS key, value FROM src limit 5
+'python newline.py' AS key, value FROM src limit 6
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src
 POSTHOOK: Output: default@tmp_tmp
@@ -31,9 +31,270 @@ POSTHOOK: Lineage: tmp_tmp.value SCRIPT 
 2	NULL
 1
 2	NULL
+1	2
 1
 2	NULL
 1
 2	NULL
-1
-2	NULL
+1	2
+PREHOOK: query: drop table tmp_tmp
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@tmp_tmp
+PREHOOK: Output: default@tmp_tmp
+POSTHOOK: query: drop table tmp_tmp
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@tmp_tmp
+POSTHOOK: Output: default@tmp_tmp
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: create table tmp_tmp(key string, value string) stored as rcfile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table tmp_tmp(key string, value string) stored as rcfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tmp_tmp
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapednewline.py' AS key, value FROM src limit 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@tmp_tmp
+POSTHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapednewline.py' AS key, value FROM src limit 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tmp_tmp
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: select * from tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\n2	NULL
+1\n2	NULL
+1\n2	NULL
+1\n2	NULL
+1\n2	NULL
+PREHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\n2	NULL
+1\n2	NULL
+1\n2	NULL
+1\n2	NULL
+1\n2	NULL
+PREHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapedcarriagereturn.py' AS key, value FROM src limit 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@tmp_tmp
+POSTHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapedcarriagereturn.py' AS key, value FROM src limit 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tmp_tmp
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: select * from tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\r2	NULL
+1\r2	NULL
+1\r2	NULL
+1\r2	NULL
+1\r2	NULL
+PREHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\r2	NULL
+1\r2	NULL
+1\r2	NULL
+1\r2	NULL
+1\r2	NULL
+PREHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapedtab.py' AS key, value FROM src limit 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@tmp_tmp
+POSTHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapedtab.py' AS key, value FROM src limit 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tmp_tmp
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: select * from tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\t2	NULL
+1\t2	NULL
+1\t2	NULL
+1\t2	NULL
+1\t2	NULL
+PREHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\t2	NULL
+1\t2	NULL
+1\t2	NULL
+1\t2	NULL
+1\t2	NULL
+PREHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python doubleescapedtab.py' AS key, value FROM src limit 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@tmp_tmp
+POSTHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python doubleescapedtab.py' AS key, value FROM src limit 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tmp_tmp
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: select * from tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\	2
+1\\t2	NULL
+1\	2
+1\\t2	NULL
+1\	2
+PREHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default),
(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\	2
+1\\t2	NULL
+1\	2
+1\\t2	NULL
+1\	2



Mime
View raw message