hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject hive git commit: HIVE-14655: LLAP input format should escape the query string being passed to getSplits() (Jason Dere, reviewed by Prasanth Jayachandran)
Date Tue, 06 Sep 2016 17:16:41 GMT
Repository: hive
Updated Branches:
  refs/heads/master e870ae1d6 -> 4cc783e55


HIVE-14655: LLAP input format should escape the query string being passed to getSplits() (Jason
Dere, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4cc783e5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4cc783e5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4cc783e5

Branch: refs/heads/master
Commit: 4cc783e5585979ca3835a9aa35376b8ed3f0b086
Parents: e870ae1
Author: Jason Dere <jdere@hortonworks.com>
Authored: Tue Sep 6 10:15:20 2016 -0700
Committer: Jason Dere <jdere@hortonworks.com>
Committed: Tue Sep 6 10:15:20 2016 -0700

----------------------------------------------------------------------
 .../org/apache/hive/jdbc/TestJdbcWithMiniLlap.java | 17 +++++++++++++++++
 .../hadoop/hive/llap/LlapBaseInputFormat.java      | 10 +++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/4cc783e5/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
index 4897364..de47412 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
@@ -34,6 +34,7 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -195,6 +196,22 @@ public class TestJdbcWithMiniLlap {
     assertArrayEquals(new String[] {"val_0", nonAscii}, rowCollector.rows.get(2));
   }
 
+  @Test(timeout = 60000)
+  public void testEscapedStrings() throws Exception {
+    createTestTable("testtab1");
+
+    RowCollector rowCollector = new RowCollector();
+    String expectedVal1 = "'a',\"b\",\\c\\";
+    String expectedVal2 = "multi\nline";
+    String query = "select value, '\\'a\\',\"b\",\\\\c\\\\', 'multi\\nline' from testtab1
where under_col=0";
+    int rowCount = processQuery(query, 1, rowCollector);
+    assertEquals(3, rowCount);
+
+    assertArrayEquals(new String[] {"val_0", expectedVal1, expectedVal2}, rowCollector.rows.get(0));
+    assertArrayEquals(new String[] {"val_0", expectedVal1, expectedVal2}, rowCollector.rows.get(1));
+    assertArrayEquals(new String[] {"val_0", expectedVal1, expectedVal2}, rowCollector.rows.get(2));
+  }
+
   private interface RowProcessor {
     void process(Row row);
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/4cc783e5/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
----------------------------------------------------------------------
diff --git a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
index c2fca54..7dae4fc 100644
--- a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
+++ b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -207,7 +208,8 @@ public class LlapBaseInputFormat<V extends WritableComparable<?>>
       throw new IOException(e);
     }
 
-    String sql = String.format(SPLIT_QUERY, query, numSplits);
+    String escapedQuery = StringUtils.escapeString(query, ESCAPE_CHAR, escapedChars);
+    String sql = String.format(SPLIT_QUERY, escapedQuery, numSplits);
     try (
       Connection con = DriverManager.getConnection(url,user,pwd);
       Statement stmt = con.createStatement();
@@ -342,6 +344,12 @@ public class LlapBaseInputFormat<V extends WritableComparable<?>>
     return ByteBuffer.wrap(containerTokens_dob.getData(), 0, containerTokens_dob.getLength());
   }
 
+  private static final char ESCAPE_CHAR = '\\';
+
+  private static final char[] escapedChars = {
+    '"', ESCAPE_CHAR
+  };
+
   private static class LlapRecordReaderTaskUmbilicalExternalResponder implements LlapTaskUmbilicalExternalResponder
{
     protected LlapBaseRecordReader<?> recordReader = null;
     protected LinkedBlockingQueue<ReaderEvent> queuedEvents = new LinkedBlockingQueue<ReaderEvent>();


Mime
View raw message