kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jdcry...@apache.org
Subject incubator-kudu git commit: KUDU-1360. [spark] KuduRelation.getKuduValue doesn't check nulls
Date Wed, 02 Mar 2016 03:46:47 GMT
Repository: incubator-kudu
Updated Branches:
  refs/heads/master 8eec83f95 -> dd6dbda35


KUDU-1360. [spark] KuduRelation.getKuduValue doesn't check nulls

Fixes the bug and adds a small test.

We should have a "all types all options" kind of test.

Change-Id: I8bfd3144806b0b84bea00e8eec77a6ba7698379b
Reviewed-on: http://gerrit.cloudera.org:8080/2381
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon <todd@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/dd6dbda3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/dd6dbda3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/dd6dbda3

Branch: refs/heads/master
Commit: dd6dbda35246722dd28e565252fd4f23c2c8dcad
Parents: 8eec83f
Author: Jean-Daniel Cryans <jdcryans@apache.org>
Authored: Tue Mar 1 16:26:58 2016 -0800
Committer: Jean-Daniel Cryans <jdcryans@gerrit.cloudera.org>
Committed: Wed Mar 2 03:42:17 2016 +0000

----------------------------------------------------------------------
 .../src/main/scala/org/kududb/spark/DefaultSource.scala | 12 +++++++++---
 .../test/scala/org/kududb/spark/DefaultSourceTest.scala |  6 +++++-
 .../src/test/scala/org/kududb/spark/TestContext.scala   | 11 +++++++++--
 3 files changed, 23 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/dd6dbda3/java/kudu-spark/src/main/scala/org/kududb/spark/DefaultSource.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/kududb/spark/DefaultSource.scala b/java/kudu-spark/src/main/scala/org/kududb/spark/DefaultSource.scala
index 50cb9c6..447b059 100644
--- a/java/kudu-spark/src/main/scala/org/kududb/spark/DefaultSource.scala
+++ b/java/kudu-spark/src/main/scala/org/kududb/spark/DefaultSource.scala
@@ -116,7 +116,7 @@ class KuduRelation(val tableName: String,
   /**
     * Build the RDD to scan rows.
     *
-    * @param requiredColumns clumns that are being requested by the requesting query
+    * @param requiredColumns columns that are being requested by the requesting query
     * @param filters         filters that are being applied by the requesting query
     * @return                RDD will all the results from Kudu
     */
@@ -128,8 +128,14 @@ class KuduRelation(val tableName: String,
   }
 
   private def getKuduValue(row: RowResult, columnName: String): Any = {
-    val columnType = kuduSchemaColumnMap.getOrElse(columnName,
-      throw new IllegalArgumentException(s"Couldn't find column '$columnName'")).getType
+    val columnSchema = kuduSchemaColumnMap.getOrElse(columnName,
+      throw new IllegalArgumentException(s"Couldn't find column '$columnName'"))
+
+    if (columnSchema.isNullable && row.isNull(columnName)) {
+      return null
+    }
+
+    val columnType = columnSchema.getType
 
     columnType match {
       case Type.BINARY => row.getBinary(columnName)

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/dd6dbda3/java/kudu-spark/src/test/scala/org/kududb/spark/DefaultSourceTest.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/kududb/spark/DefaultSourceTest.scala b/java/kudu-spark/src/test/scala/org/kududb/spark/DefaultSourceTest.scala
index 17cea07..483b2e2 100644
--- a/java/kudu-spark/src/test/scala/org/kududb/spark/DefaultSourceTest.scala
+++ b/java/kudu-spark/src/test/scala/org/kududb/spark/DefaultSourceTest.scala
@@ -35,6 +35,10 @@ class DefaultSourceTest extends FunSuite with TestContext {
       Map("kudu.table" -> tableName, "kudu.master" -> miniCluster.getMasterAddresses))
       .registerTempTable(tableName)
 
-    assert(sqlContext.sql("SELECT * FROM " + tableName).collectAsList().size() == rowCount)
+    val results = sqlContext.sql("SELECT * FROM " + tableName).collectAsList()
+    assert(results.size() == rowCount)
+
+    assert(results.get(0).isNullAt(2))
+    assert(!results.get(1).isNullAt(2))
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/dd6dbda3/java/kudu-spark/src/test/scala/org/kududb/spark/TestContext.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/kududb/spark/TestContext.scala b/java/kudu-spark/src/test/scala/org/kududb/spark/TestContext.scala
index fa24f52..9876282 100644
--- a/java/kudu-spark/src/test/scala/org/kududb/spark/TestContext.scala
+++ b/java/kudu-spark/src/test/scala/org/kududb/spark/TestContext.scala
@@ -39,7 +39,7 @@ trait TestContext extends BeforeAndAfterAll { self: Suite =>
     val columns = ImmutableList.of(
       new ColumnSchemaBuilder("key", Type.INT32).key(true).build(),
       new ColumnSchemaBuilder("c1_i", Type.INT32).build(),
-      new ColumnSchemaBuilder("c2_s", Type.STRING).build())
+      new ColumnSchemaBuilder("c2_s", Type.STRING).nullable(true).build())
     new Schema(columns)
   }
 
@@ -75,7 +75,14 @@ trait TestContext extends BeforeAndAfterAll { self: Suite =>
       val row = insert.getRow
       row.addInt(0, i)
       row.addInt(1, i)
-      row.addString(2, i.toString)
+
+      // Sprinkling some nulls so that queries see them.
+      if (i % 2 == 0) {
+        row.addString(2, i.toString)
+      } else {
+        row.setNull(2)
+      }
+
       kuduSession.apply(insert)
     }
   }


Mime
View raw message