drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amansi...@apache.org
Subject [1/6] drill git commit: DRILL-3250: Drill fails to compare multi-byte characters from hive table - A small refactoring of original fix of this issue (DRILL-4039); - Added test for the fix.
Date Sat, 20 May 2017 14:26:00 GMT
Repository: drill
Updated Branches:
  refs/heads/master 0dc237e31 -> 416ec70a6


DRILL-3250: Drill fails to compare multi-byte characters from hive table - A small refactoring
of original fix of this issue (DRILL-4039); - Added test for the fix.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2988f716
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2988f716
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2988f716

Branch: refs/heads/master
Commit: 2988f716f9a43d93dd3968b5c22a24a76bbd708e
Parents: 0dc237e
Author: Vitalii Diravka <vitalii.diravka@gmail.com>
Authored: Tue May 16 09:53:08 2017 +0000
Committer: Aman Sinha <asinha@maprtech.com>
Committed: Fri May 19 09:49:09 2017 -0700

----------------------------------------------------------------------
 .../exec/store/hive/schema/DrillHiveTable.java  | 13 +++++-----
 .../apache/drill/exec/hive/TestHiveStorage.java | 25 ++++++++++++++++++++
 .../sql/handlers/DescribeTableHandler.java      | 10 ++++----
 .../planner/sql/handlers/ShowTablesHandler.java |  7 +++---
 4 files changed, 40 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/2988f716/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
index 1e65ee0..93dbc9c 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -17,8 +17,7 @@
  */
 package org.apache.drill.exec.store.hive.schema;
 
-import java.nio.charset.Charset;
-import org.apache.drill.exec.planner.sql.parser.DrillParserUtil;
+import org.apache.calcite.util.Util;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -118,7 +117,7 @@ public class DrillHiveTable extends DrillTable{
         int maxLen = TypeInfoUtils.getCharacterLengthForType(pTypeInfo);
         return typeFactory.createTypeWithCharsetAndCollation(
           typeFactory.createSqlType(SqlTypeName.VARCHAR, maxLen), /*input type*/
-          Charset.forName(DrillParserUtil.CHARSET),
+          Util.getDefaultCharset(),
           SqlCollation.IMPLICIT /* TODO: need to decide if implicit is the correct one */
         );
       }
@@ -126,9 +125,9 @@ public class DrillHiveTable extends DrillTable{
       case CHAR: {
         int maxLen = TypeInfoUtils.getCharacterLengthForType(pTypeInfo);
         return typeFactory.createTypeWithCharsetAndCollation(
-            typeFactory.createSqlType(SqlTypeName.CHAR, maxLen), /*input type*/
-            Charset.forName(DrillParserUtil.CHARSET),
-            SqlCollation.IMPLICIT
+          typeFactory.createSqlType(SqlTypeName.CHAR, maxLen), /*input type*/
+          Util.getDefaultCharset(),
+          SqlCollation.IMPLICIT
         );
       }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/2988f716/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
index e00314d..082a019 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
@@ -17,8 +17,13 @@
  */
 package org.apache.drill.exec.hive;
 
+import mockit.Mock;
+import mockit.MockUp;
+import mockit.integration.junit4.JMockit;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
+import org.apache.calcite.util.Util;
+import org.apache.calcite.util.ConversionUtil;
 import org.apache.drill.common.exceptions.UserRemoteException;
 import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.planner.physical.PlannerSettings;
@@ -29,8 +34,10 @@ import org.joda.time.DateTime;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
 
 import java.math.BigDecimal;
+import java.nio.charset.Charset;
 import java.sql.Date;
 import java.sql.Timestamp;
 import java.util.List;
@@ -43,6 +50,7 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
+@RunWith(JMockit.class)
 public class TestHiveStorage extends HiveTestBase {
   @BeforeClass
   public static void setupOptions() throws Exception {
@@ -538,6 +546,23 @@ public class TestHiveStorage extends HiveTestBase {
     }
   }
 
+  @Test // DRILL-3250
+  public void testNonAsciiStringLiterals() throws Exception {
+    // mock calcite util method to return utf charset
+    // instead of setting saffron.default.charset at system level
+    new MockUp<Util>() {
+      @Mock
+      Charset getDefaultCharset() {
+        return Charset.forName(ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
+      }
+    };
+
+    testBuilder()
+        .sqlQuery("select * from hive.empty_table where b = 'Абвгде谢谢'")
+        .expectsEmptyResultSet()
+        .go();
+  }
+
   private void verifyColumnsMetadata(List<UserProtos.ResultColumnMetadata> columnsList,
Map<String, Integer> expectedResult) {
     for (UserProtos.ResultColumnMetadata columnMetadata : columnsList) {
       assertTrue("Column should be present in result set", expectedResult.containsKey(columnMetadata.getColumnName()));

http://git-wip-us.apache.org/repos/asf/drill/blob/2988f716/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DescribeTableHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DescribeTableHandler.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DescribeTableHandler.java
index 9c14c59..72b1aef 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DescribeTableHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DescribeTableHandler.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -18,7 +18,6 @@
 
 package org.apache.drill.exec.planner.sql.handlers;
 
-import static org.apache.drill.exec.planner.sql.parser.DrillParserUtil.CHARSET;
 import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.COLS_COL_COLUMN_NAME;
 import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.COLS_COL_DATA_TYPE;
 import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.COLS_COL_IS_NULLABLE;
@@ -70,6 +69,7 @@ public class DescribeTableHandler extends DefaultSqlHandler {
       final SchemaPlus defaultSchema = config.getConverter().getDefaultSchema();
       final List<String> schemaPathGivenInCmd = Util.skipLast(table.names);
       final SchemaPlus schema = SchemaUtilites.findSchema(defaultSchema, schemaPathGivenInCmd);
+      final String charset = Util.getDefaultCharset().name();
 
       if (schema == null) {
         SchemaUtilites.throwSchemaNotFoundException(defaultSchema,
@@ -98,14 +98,14 @@ public class DescribeTableHandler extends DefaultSqlHandler {
         schemaCondition = DrillParserUtil.createCondition(
             new SqlIdentifier(SHRD_COL_TABLE_SCHEMA, SqlParserPos.ZERO),
             SqlStdOperatorTable.EQUALS,
-            SqlLiteral.createCharString(schemaPath, CHARSET, SqlParserPos.ZERO)
+            SqlLiteral.createCharString(schemaPath, charset, SqlParserPos.ZERO)
         );
       }
 
       SqlNode where = DrillParserUtil.createCondition(
           new SqlIdentifier(SHRD_COL_TABLE_NAME, SqlParserPos.ZERO),
           SqlStdOperatorTable.EQUALS,
-          SqlLiteral.createCharString(tableName, CHARSET, SqlParserPos.ZERO));
+          SqlLiteral.createCharString(tableName, charset, SqlParserPos.ZERO));
 
       where = DrillParserUtil.createCondition(schemaCondition, SqlStdOperatorTable.AND, where);
 
@@ -115,7 +115,7 @@ public class DescribeTableHandler extends DefaultSqlHandler {
             DrillParserUtil.createCondition(
                 new SqlIdentifier(COLS_COL_COLUMN_NAME, SqlParserPos.ZERO),
                 SqlStdOperatorTable.EQUALS,
-                SqlLiteral.createCharString(node.getColumn().toString(), CHARSET, SqlParserPos.ZERO));
+                SqlLiteral.createCharString(node.getColumn().toString(), charset, SqlParserPos.ZERO));
       } else if (node.getColumnQualifier() != null) {
         columnFilter =
             DrillParserUtil.createCondition(

http://git-wip-us.apache.org/repos/asf/drill/blob/2988f716/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowTablesHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowTablesHandler.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowTablesHandler.java
index 206f966..ada7c3a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowTablesHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowTablesHandler.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -18,7 +18,6 @@
 
 package org.apache.drill.exec.planner.sql.handlers;
 
-import static org.apache.drill.exec.planner.sql.parser.DrillParserUtil.CHARSET;
 import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.IS_SCHEMA_NAME;
 import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.SHRD_COL_TABLE_NAME;
 import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.SHRD_COL_TABLE_SCHEMA;
@@ -35,6 +34,7 @@ import org.apache.calcite.sql.SqlSelect;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.sql.parser.SqlParserPos;
 import org.apache.calcite.tools.RelConversionException;
+import org.apache.calcite.util.Util;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.exec.planner.sql.SchemaUtilites;
 import org.apache.drill.exec.planner.sql.parser.DrillParserUtil;
@@ -83,10 +83,11 @@ public class ShowTablesHandler extends DefaultSqlHandler {
       tableSchema = drillSchema.getFullSchemaName();
     }
 
+    final String charset = Util.getDefaultCharset().name();
     where = DrillParserUtil.createCondition(
         new SqlIdentifier(SHRD_COL_TABLE_SCHEMA, SqlParserPos.ZERO),
         SqlStdOperatorTable.EQUALS,
-        SqlLiteral.createCharString(tableSchema, CHARSET, SqlParserPos.ZERO));
+        SqlLiteral.createCharString(tableSchema, charset, SqlParserPos.ZERO));
 
     SqlNode filter = null;
     final SqlNode likePattern = node.getLikePattern();


Mime
View raw message