impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tarmstr...@apache.org
Subject [1/2] impala git commit: IMPALA-6371: Additional check for delimiters
Date Thu, 11 Jan 2018 01:25:21 GMT
Repository: impala
Updated Branches:
  refs/heads/master 3394eb1be -> c0c1202db


IMPALA-6371: Additional check for delimiters

The check validates the codepoint of the Java char.

Testing:
- Added tests for valid/invalid unicode in HdfsStorageDescriptorTest.

Change-Id: If8dc335d39dd02f602cf93682bccf84b2c099dde
Reviewed-on: http://gerrit.cloudera.org:8080/8959
Reviewed-by: Alex Behm <alex.behm@cloudera.com>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/a34df684
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/a34df684
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/a34df684

Branch: refs/heads/master
Commit: a34df684f78ede6925ebeff1ed174b00828a980f
Parents: 3394eb1
Author: Adam Holley <aholley@cloudera.com>
Authored: Sun Jan 7 19:13:31 2018 -0600
Committer: Impala Public Jenkins <impala-public-jenkins@gerrit.cloudera.org>
Committed: Wed Jan 10 22:31:32 2018 +0000

----------------------------------------------------------------------
 .../impala/catalog/HdfsStorageDescriptor.java     |  7 ++++++-
 .../impala/catalog/HdfsStorageDescriptorTest.java | 18 ++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/impala/blob/a34df684/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java b/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
index 4554a31..b4e2564 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
@@ -140,7 +140,12 @@ public class HdfsStorageDescriptor {
       // need support from the Hive side.
       return Byte.parseByte(delimVal);
     } catch (NumberFormatException e) {
-      if (delimVal.length() == 1) return (byte) delimVal.charAt(0);
+      if (delimVal.length() == 1) {
+        // Adding additional check as Java chars are two bytes.
+        // e.g. \u1111 as delimVal will return a valid byte '11'
+        int cp = Character.codePointAt(delimVal, 0);
+        if (cp >= 0 && cp <= 255) return (byte) cp;
+      }
     }
     return null;
   }

http://git-wip-us.apache.org/repos/asf/impala/blob/a34df684/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java b/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java
index 0934fed..bb133a1 100644
--- a/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java
@@ -157,5 +157,23 @@ public class HdfsStorageDescriptorTest {
           "single character or as a decimal value in the range [-128:127]",
           e.getMessage());
     }
+
+    // Test that a unicode character out of the valid range will not be accepted.
+    sd.getSerdeInfo().setParameters(new HashMap<String,String>());
+    sd.getSerdeInfo().putToParameters(serdeConstants.LINE_DELIM, "\u1111");
+    try {
+      HdfsStorageDescriptor.fromStorageDescriptor("fake", sd);
+      fail();
+    } catch (HdfsStorageDescriptor.InvalidStorageDescriptorException e) {
+      assertEquals("Invalid delimiter: '\u1111'. Delimiter must be specified as a " +
+          "single character or as a decimal value in the range [-128:127]",
+          e.getMessage());
+    }
+
+    // Validate that unicode character in the valid range will be accepted.
+    sd.getSerdeInfo().setParameters(new HashMap<String,String>());
+    sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM, "\u0001");
+    assertNotNull(HdfsStorageDescriptor.fromStorageDescriptor("fakeTbl", sd));
+
   }
 }


Mime
View raw message