drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ve...@apache.org
Subject [3/3] drill git commit: DRILL-3266: Add hive-contrib dependency to Hive storage plugin
Date Wed, 17 Jun 2015 21:31:06 GMT
DRILL-3266: Add hive-contrib dependency to Hive storage plugin

Module hive-contrib has SerDes and UDF implementations which we want them to
be available by default in Drill Hive storage plugin.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/d32acdb9
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/d32acdb9
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/d32acdb9

Branch: refs/heads/master
Commit: d32acdb9020aaaba656588a03a02dd42e6a61241
Parents: b46c53e
Author: vkorukanti <venki.korukanti@gmail.com>
Authored: Mon Jun 15 08:11:10 2015 -0700
Committer: vkorukanti <venki.korukanti@gmail.com>
Committed: Wed Jun 17 14:25:35 2015 -0700

----------------------------------------------------------------------
 contrib/storage-hive/core/pom.xml               |  4 +++
 .../apache/drill/exec/hive/TestHiveStorage.java | 10 ++++++++
 .../exec/hive/TestInfoSchemaOnHiveStorage.java  | 20 ++++++++++++---
 .../exec/store/hive/HiveTestDataGenerator.java  | 24 +++++++++--------
 pom.xml                                         | 27 ++++++++++++++++++++
 5 files changed, 71 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/contrib/storage-hive/core/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml
index fb96a0d..546fd7b 100644
--- a/contrib/storage-hive/core/pom.xml
+++ b/contrib/storage-hive/core/pom.xml
@@ -61,6 +61,10 @@
       <artifactId>hive-hbase-handler</artifactId>
     </dependency>
     <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-contrib</artifactId>
+    </dependency>
+    <dependency>
       <groupId>org.apache.calcite</groupId>
       <artifactId>calcite-core</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
index 2898f91..39e8f54 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
@@ -186,4 +186,14 @@ public class TestHiveStorage extends HiveTestBase {
         .baselineValues(5, " key_5")
         .go();
   }
+
+  @Test // DRILL-3266
+  public void queryingTableWithSerDeInHiveContribJar() throws Exception {
+    testBuilder()
+        .sqlQuery("SELECT * FROM hive.db1.kv_db1 ORDER BY key DESC LIMIT 1")
+        .unOrdered()
+        .baselineColumns("key", "value")
+        .baselineValues("5", " key_5")
+        .go();
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
index b684538..6118be5 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
@@ -93,7 +93,13 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase {
   // When table name is fully qualified with schema name (sub-schema is non-default schema)
   @Test
   public void describeTable2() throws Exception{
-    describeHelper(null, "DESCRIBE hive.`db1`.kv_db1");
+    testBuilder()
+        .sqlQuery("DESCRIBE hive.`db1`.kv_db1")
+        .unOrdered()
+        .baselineColumns(baselineCols)
+        .baselineValues("key", "CHARACTER VARYING", "YES")
+        .baselineValues("value", "CHARACTER VARYING", "YES")
+        .go();
   }
 
   // When table is qualified with just the top level schema. It should look for the table
in default sub-schema within
@@ -113,7 +119,13 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase {
   // given as single level schema name.
   @Test
   public void describeTable5() throws Exception {
-    describeHelper(null, "DESCRIBE `hive.db1`.kv_db1");
+    testBuilder()
+        .sqlQuery("DESCRIBE `hive.db1`.kv_db1")
+        .unOrdered()
+        .baselineColumns(baselineCols)
+        .baselineValues("key", "CHARACTER VARYING", "YES")
+        .baselineValues("value", "CHARACTER VARYING", "YES")
+        .go();
   }
 
   // When current default schema is just the top-level schema name and the table has no schema
qualifier. It should
@@ -187,8 +199,8 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase {
         .unOrdered()
         .optionSettingQueriesForTestQuery("USE hive.db1")
         .baselineColumns("key", "value")
-        .baselineValues(1, " key_1")
-        .baselineValues(2, " key_2")
+        .baselineValues("1", " key_1")
+        .baselineValues("2", " key_2")
         .go();
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index 5b60c05..ea8d90f 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -130,9 +130,21 @@ public class HiveTestDataGenerator {
     // generate (key, value) test data
     String testDataFile = generateTestDataFile();
 
-    createTableAndLoadData(hiveDriver, "default", "kv", testDataFile);
+    // Create a (key, value) schema table with Text SerDe which is available in hive-serdes.jar
+    executeQuery(hiveDriver, "CREATE TABLE IF NOT EXISTS default.kv(key INT, value STRING)
" +
+        "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE");
+    executeQuery(hiveDriver, "LOAD DATA LOCAL INPATH '" + testDataFile + "' OVERWRITE INTO
TABLE default.kv");
+
+    // Create a (key, value) schema table in non-default database with RegexSerDe which is
available in hive-contrib.jar
+    // Table with RegExSerde is expected to have columns of STRING type only.
     executeQuery(hiveDriver, "CREATE DATABASE IF NOT EXISTS db1");
-    createTableAndLoadData(hiveDriver, "db1", "kv_db1", testDataFile);
+    executeQuery(hiveDriver, "CREATE TABLE db1.kv_db1(key STRING, value STRING) " +
+        "ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.RegexSerDe' " +
+        "WITH SERDEPROPERTIES (" +
+        "  \"input.regex\" = \"([0-9]*), (.*_[0-9]*)\", " +
+        "  \"output.format.string\" = \"%1$s, %2$s\"" +
+        ") ");
+    executeQuery(hiveDriver, "INSERT INTO TABLE db1.kv_db1 SELECT * FROM default.kv");
 
     // Create an Avro format based table backed by schema in a separate file
     final String avroCreateQuery = String.format("CREATE TABLE db1.avro " +
@@ -293,14 +305,6 @@ public class HiveTestDataGenerator {
     ss.close();
   }
 
-  private void createTableAndLoadData(Driver hiveDriver, String dbName, String tblName, String
dataFile) {
-    executeQuery(hiveDriver, String.format("USE %s", dbName));
-    executeQuery(hiveDriver, String.format("CREATE TABLE IF NOT EXISTS %s.%s(key INT, value
STRING) "+
-        "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE", dbName, tblName));
-    executeQuery(hiveDriver,
-        String.format("LOAD DATA LOCAL INPATH '%s' OVERWRITE INTO TABLE %s.%s", dataFile,
dbName, tblName));
-  }
-
   private File getTempFile() throws Exception {
     return java.nio.file.Files.createTempFile("drill-hive-test", ".txt").toFile();
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1b14ae1..675a6e0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -711,6 +711,33 @@
           </exclusion>
         </exclusions>
       </dependency>
+      <dependency>
+        <groupId>org.apache.hive</groupId>
+        <artifactId>hive-contrib</artifactId>
+        <version>${hive.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>org.apache.hive</groupId>
+            <artifactId>hive-serde</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.hive</groupId>
+            <artifactId>hive-shims</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.hive</groupId>
+            <artifactId>hive-exec</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+          </exclusion>
+          <exclusion>
+            <artifactId>slf4j-log4j12</artifactId>
+            <groupId>org.slf4j</groupId>
+          </exclusion>
+        </exclusions>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 


Mime
View raw message