spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wenc...@apache.org
Subject [spark] branch master updated: [SPARK-27174][SQL] Add support for casting integer types to binary
Date Fri, 22 Mar 2019 17:10:28 GMT
This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 8efc5ec  [SPARK-27174][SQL] Add support for casting integer types to binary
8efc5ec is described below

commit 8efc5ec72e2f5899547941010e22c023d6cb86b3
Author: Martin Junghanns <martin.junghanns@neotechnology.com>
AuthorDate: Fri Mar 22 10:09:35 2019 -0700

    [SPARK-27174][SQL] Add support for casting integer types to binary
    
    Co-authored-by: Philip Stutz <philip.stutzgmail.com>
    
    ## What changes were proposed in this pull request?
    
    This PR adds support for casting
    
    * `ByteType`
    * `ShortType`
    * `IntegerType`
    * `LongType`
    
    to `BinaryType`.
    
    ## How was this patch tested?
    
    We added unit tests for casting instances of the above types. For validation, we used
Javas `DataOutputStream` to compare the resulting byte array with the result of `Cast`.
    
    We state that the contribution is our original work and that we license the work to the
project under the project’s open source license.
    
    cloud-fan we'd appreciate a review if you find the time, thx
    
    Closes #24107 from s1ck/cast_to_binary.
    
    Authored-by: Martin Junghanns <martin.junghanns@neotechnology.com>
    Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/expressions/Cast.scala      |  11 +-
 .../spark/sql/catalyst/util/NumberConverter.scala  |  35 ++++++
 .../sql/catalyst/analysis/AnalysisErrorSuite.scala |   5 -
 .../sql/catalyst/util/NumberConverterSuite.scala   |  48 ++++++++-
 .../src/test/resources/sql-tests/inputs/cast.sql   |  13 +++
 .../test/resources/sql-tests/results/cast.sql.out  |  84 +++++++++++++--
 .../typeCoercion/native/binaryComparison.sql.out   | 120 +++++++++------------
 .../native/windowFrameCoercion.sql.out             |   2 +-
 .../scala/org/apache/spark/sql/DatasetSuite.scala  |   4 +-
 9 files changed, 234 insertions(+), 88 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 72cb6b2..848195f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -44,6 +44,7 @@ object Cast {
     case (_, StringType) => true
 
     case (StringType, BinaryType) => true
+    case (_: IntegralType, BinaryType) => true
 
     case (StringType, BooleanType) => true
     case (DateType, BooleanType) => true
@@ -326,6 +327,10 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
   // BinaryConverter
   private[this] def castToBinary(from: DataType): Any => Any = from match {
     case StringType => buildCast[UTF8String](_, _.getBytes)
+    case ByteType => buildCast[Byte](_, NumberConverter.toBinary)
+    case ShortType => buildCast[Short](_, NumberConverter.toBinary)
+    case IntegerType => buildCast[Int](_, NumberConverter.toBinary)
+    case LongType => buildCast[Long](_, NumberConverter.toBinary)
   }
 
   // UDFToBoolean
@@ -908,7 +913,11 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
 
   private[this] def castToBinaryCode(from: DataType): CastFunction = from match {
     case StringType =>
-      (c, evPrim, evNull) => code"$evPrim = $c.getBytes();"
+      (c, evPrim, evNull) =>
+        code"$evPrim = $c.getBytes();"
+    case _: IntegralType =>
+      (c, evPrim, evNull) =>
+        code"$evPrim = ${NumberConverter.getClass.getName.stripSuffix("$")}.toBinary($c);"
   }
 
   private[this] def castToDateCode(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala
index 9c3f6b7..7dbdd1e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala
@@ -169,4 +169,39 @@ object NumberConverter {
     }
     UTF8String.fromBytes(java.util.Arrays.copyOfRange(temp, resultStartPos, temp.length))
   }
+
+  def toBinary(l: Long): Array[Byte] = {
+    val result = new Array[Byte](8)
+    result(0) = (l >>> 56 & 0xFF).toByte
+    result(1) = (l >>> 48 & 0xFF).toByte
+    result(2) = (l >>> 40 & 0xFF).toByte
+    result(3) = (l >>> 32 & 0xFF).toByte
+    result(4) = (l >>> 24 & 0xFF).toByte
+    result(5) = (l >>> 16 & 0xFF).toByte
+    result(6) = (l >>> 8 & 0xFF).toByte
+    result(7) = (l & 0xFF).toByte
+    result
+  }
+
+  def toBinary(i: Int): Array[Byte] = {
+    val result = new Array[Byte](4)
+    result(0) = (i >>> 24 & 0xFF).toByte
+    result(1) = (i >>> 16 & 0xFF).toByte
+    result(2) = (i >>> 8 & 0xFF).toByte
+    result(3) = (i & 0xFF).toByte
+    result
+  }
+
+  def toBinary(s: Short): Array[Byte] = {
+    val result = new Array[Byte](2)
+    result(0) = (s >>> 8 & 0xFF).toByte
+    result(1) = (s & 0xFF).toByte
+    result
+  }
+
+  def toBinary(s: Byte): Array[Byte] = {
+    val result = new Array[Byte](1)
+    result(0) = s
+    result
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 129ce3b..ec71c1c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -216,11 +216,6 @@ class AnalysisErrorSuite extends AnalysisTest {
     "Invalid usage of '*'" :: "in expression 'sum'" :: Nil)
 
   errorTest(
-    "bad casts",
-    testRelation.select(Literal(1).cast(BinaryType).as('badCast)),
-  "cannot cast" :: Literal(1).dataType.simpleString :: BinaryType.simpleString :: Nil)
-
-  errorTest(
     "sorting by unsupported column types",
     mapRelation.orderBy('map.asc),
     "sort" :: "type" :: "map<int,int>" :: Nil)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala
index 13265a1..ec73f45 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala
@@ -17,8 +17,11 @@
 
 package org.apache.spark.sql.catalyst.util
 
+import java.nio.ByteBuffer
+import java.nio.ByteOrder.BIG_ENDIAN
+
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.util.NumberConverter.convert
+import org.apache.spark.sql.catalyst.util.NumberConverter.{convert, toBinary}
 import org.apache.spark.unsafe.types.UTF8String
 
 class NumberConverterSuite extends SparkFunSuite {
@@ -37,4 +40,47 @@ class NumberConverterSuite extends SparkFunSuite {
     checkConv("11abc", 10, 16, "B")
   }
 
+  test("byte to binary") {
+    checkToBinary(0.toByte)
+    checkToBinary(1.toByte)
+    checkToBinary(-1.toByte)
+    checkToBinary(Byte.MaxValue)
+    checkToBinary(Byte.MinValue)
+  }
+
+  test("short to binary") {
+    checkToBinary(0.toShort)
+    checkToBinary(1.toShort)
+    checkToBinary(-1.toShort)
+    checkToBinary(Short.MaxValue)
+    checkToBinary(Short.MinValue)
+  }
+
+  test("integer to binary") {
+    checkToBinary(0)
+    checkToBinary(1)
+    checkToBinary(-1)
+    checkToBinary(Int.MaxValue)
+    checkToBinary(Int.MinValue)
+  }
+
+  test("long to binary") {
+    checkToBinary(0L)
+    checkToBinary(1L)
+    checkToBinary(-1L)
+    checkToBinary(Long.MaxValue)
+    checkToBinary(Long.MinValue)
+  }
+
+  def checkToBinary[T](in: T): Unit = in match {
+    case b: Byte =>
+      assert(toBinary(b) === ByteBuffer.allocate(1).order(BIG_ENDIAN).put(b).array())
+    case s: Short =>
+      assert(toBinary(s) === ByteBuffer.allocate(2).order(BIG_ENDIAN).putShort(s).array())
+    case i: Int =>
+      assert(toBinary(i) === ByteBuffer.allocate(4).order(BIG_ENDIAN).putInt(i).array())
+    case l: Long =>
+      assert(toBinary(l) === ByteBuffer.allocate(8).order(BIG_ENDIAN).putLong(l).array())
+  }
+
 }
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cast.sql b/sql/core/src/test/resources/sql-tests/inputs/cast.sql
index 629df59..7244cd3 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/cast.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/cast.sql
@@ -40,6 +40,19 @@ SELECT CAST('-9223372036854775809' AS long);
 SELECT CAST('9223372036854775807' AS long);
 SELECT CAST('9223372036854775808' AS long);
 
+-- cast string to its binary representation
+SELECT HEX(CAST('abc' AS binary));
+
+-- cast integral values to their corresponding binary representation
+SELECT HEX(CAST(CAST(123 AS byte) AS binary));
+SELECT HEX(CAST(CAST(-123 AS byte) AS binary));
+SELECT HEX(CAST(123S AS binary));
+SELECT HEX(CAST(-123S AS binary));
+SELECT HEX(CAST(123 AS binary));
+SELECT HEX(CAST(-123 AS binary));
+SELECT HEX(CAST(123L AS binary));
+SELECT HEX(CAST(-123L AS binary));
+
 DESC FUNCTION boolean;
 DESC FUNCTION EXTENDED boolean;
 -- TODO: migrate all cast tests here.
diff --git a/sql/core/src/test/resources/sql-tests/results/cast.sql.out b/sql/core/src/test/resources/sql-tests/results/cast.sql.out
index 9c5f455..21b18e9 100644
--- a/sql/core/src/test/resources/sql-tests/results/cast.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cast.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 24
+-- Number of queries: 33
 
 
 -- !query 0
@@ -179,20 +179,92 @@ NULL
 
 
 -- !query 22
-DESC FUNCTION boolean
+SELECT HEX(CAST('abc' AS binary))
 -- !query 22 schema
-struct<function_desc:string>
+struct<hex(CAST(abc AS BINARY)):string>
 -- !query 22 output
+616263
+
+
+-- !query 23
+SELECT HEX(CAST(CAST(123 AS byte) AS binary))
+-- !query 23 schema
+struct<hex(CAST(CAST(123 AS TINYINT) AS BINARY)):string>
+-- !query 23 output
+7B
+
+
+-- !query 24
+SELECT HEX(CAST(CAST(-123 AS byte) AS binary))
+-- !query 24 schema
+struct<hex(CAST(CAST(-123 AS TINYINT) AS BINARY)):string>
+-- !query 24 output
+85
+
+
+-- !query 25
+SELECT HEX(CAST(123S AS binary))
+-- !query 25 schema
+struct<hex(CAST(123 AS BINARY)):string>
+-- !query 25 output
+007B
+
+
+-- !query 26
+SELECT HEX(CAST(-123S AS binary))
+-- !query 26 schema
+struct<hex(CAST(-123 AS BINARY)):string>
+-- !query 26 output
+FF85
+
+
+-- !query 27
+SELECT HEX(CAST(123 AS binary))
+-- !query 27 schema
+struct<hex(CAST(123 AS BINARY)):string>
+-- !query 27 output
+0000007B
+
+
+-- !query 28
+SELECT HEX(CAST(-123 AS binary))
+-- !query 28 schema
+struct<hex(CAST(-123 AS BINARY)):string>
+-- !query 28 output
+FFFFFF85
+
+
+-- !query 29
+SELECT HEX(CAST(123L AS binary))
+-- !query 29 schema
+struct<hex(CAST(123 AS BINARY)):string>
+-- !query 29 output
+000000000000007B
+
+
+-- !query 30
+SELECT HEX(CAST(-123L AS binary))
+-- !query 30 schema
+struct<hex(CAST(-123 AS BINARY)):string>
+-- !query 30 output
+FFFFFFFFFFFFFF85
+
+
+-- !query 31
+DESC FUNCTION boolean
+-- !query 31 schema
+struct<function_desc:string>
+-- !query 31 output
 Class: org.apache.spark.sql.catalyst.expressions.Cast
 Function: boolean
 Usage: boolean(expr) - Casts the value `expr` to the target data type `boolean`.
 
 
--- !query 23
+-- !query 32
 DESC FUNCTION EXTENDED boolean
--- !query 23 schema
+-- !query 32 schema
 struct<function_desc:string>
--- !query 23 output
+-- !query 32 output
 Class: org.apache.spark.sql.catalyst.expressions.Cast
 Extended Usage:
     No example/argument for boolean.
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out
b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out
index 2914d60..b764ce4 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out
@@ -13,217 +13,193 @@ struct<>
 -- !query 1
 SELECT cast(1 as binary) = '1' FROM t
 -- !query 1 schema
-struct<>
+struct<(CAST(1 AS BINARY) = CAST(1 AS BINARY)):boolean>
 -- !query 1 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 7
+false
 
 
 -- !query 2
 SELECT cast(1 as binary) > '2' FROM t
 -- !query 2 schema
-struct<>
+struct<(CAST(1 AS BINARY) > CAST(2 AS BINARY)):boolean>
 -- !query 2 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 7
+false
 
 
 -- !query 3
 SELECT cast(1 as binary) >= '2' FROM t
 -- !query 3 schema
-struct<>
+struct<(CAST(1 AS BINARY) >= CAST(2 AS BINARY)):boolean>
 -- !query 3 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 7
+false
 
 
 -- !query 4
 SELECT cast(1 as binary) < '2' FROM t
 -- !query 4 schema
-struct<>
+struct<(CAST(1 AS BINARY) < CAST(2 AS BINARY)):boolean>
 -- !query 4 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 7
+true
 
 
 -- !query 5
 SELECT cast(1 as binary) <= '2' FROM t
 -- !query 5 schema
-struct<>
+struct<(CAST(1 AS BINARY) <= CAST(2 AS BINARY)):boolean>
 -- !query 5 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 7
+true
 
 
 -- !query 6
 SELECT cast(1 as binary) <> '2' FROM t
 -- !query 6 schema
-struct<>
+struct<(NOT (CAST(1 AS BINARY) = CAST(2 AS BINARY))):boolean>
 -- !query 6 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 7
+true
 
 
 -- !query 7
 SELECT cast(1 as binary) = cast(null as string) FROM t
 -- !query 7 schema
-struct<>
+struct<(CAST(1 AS BINARY) = CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
 -- !query 7 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 7
+NULL
 
 
 -- !query 8
 SELECT cast(1 as binary) > cast(null as string) FROM t
 -- !query 8 schema
-struct<>
+struct<(CAST(1 AS BINARY) > CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
 -- !query 8 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 7
+NULL
 
 
 -- !query 9
 SELECT cast(1 as binary) >= cast(null as string) FROM t
 -- !query 9 schema
-struct<>
+struct<(CAST(1 AS BINARY) >= CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
 -- !query 9 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 7
+NULL
 
 
 -- !query 10
 SELECT cast(1 as binary) < cast(null as string) FROM t
 -- !query 10 schema
-struct<>
+struct<(CAST(1 AS BINARY) < CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
 -- !query 10 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 7
+NULL
 
 
 -- !query 11
 SELECT cast(1 as binary) <= cast(null as string) FROM t
 -- !query 11 schema
-struct<>
+struct<(CAST(1 AS BINARY) <= CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
 -- !query 11 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 7
+NULL
 
 
 -- !query 12
 SELECT cast(1 as binary) <> cast(null as string) FROM t
 -- !query 12 schema
-struct<>
+struct<(NOT (CAST(1 AS BINARY) = CAST(CAST(NULL AS STRING) AS BINARY))):boolean>
 -- !query 12 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 7
+NULL
 
 
 -- !query 13
 SELECT '1' = cast(1 as binary) FROM t
 -- !query 13 schema
-struct<>
+struct<(CAST(1 AS BINARY) = CAST(1 AS BINARY)):boolean>
 -- !query 13 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 13
+false
 
 
 -- !query 14
 SELECT '2' > cast(1 as binary) FROM t
 -- !query 14 schema
-struct<>
+struct<(CAST(2 AS BINARY) > CAST(1 AS BINARY)):boolean>
 -- !query 14 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 13
+true
 
 
 -- !query 15
 SELECT '2' >= cast(1 as binary) FROM t
 -- !query 15 schema
-struct<>
+struct<(CAST(2 AS BINARY) >= CAST(1 AS BINARY)):boolean>
 -- !query 15 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 14
+true
 
 
 -- !query 16
 SELECT '2' < cast(1 as binary) FROM t
 -- !query 16 schema
-struct<>
+struct<(CAST(2 AS BINARY) < CAST(1 AS BINARY)):boolean>
 -- !query 16 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 13
+false
 
 
 -- !query 17
 SELECT '2' <= cast(1 as binary) FROM t
 -- !query 17 schema
-struct<>
+struct<(CAST(2 AS BINARY) <= CAST(1 AS BINARY)):boolean>
 -- !query 17 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 14
+false
 
 
 -- !query 18
 SELECT '2' <> cast(1 as binary) FROM t
 -- !query 18 schema
-struct<>
+struct<(NOT (CAST(2 AS BINARY) = CAST(1 AS BINARY))):boolean>
 -- !query 18 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 14
+true
 
 
 -- !query 19
 SELECT cast(null as string) = cast(1 as binary) FROM t
 -- !query 19 schema
-struct<>
+struct<(CAST(CAST(NULL AS STRING) AS BINARY) = CAST(1 AS BINARY)):boolean>
 -- !query 19 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 30
+NULL
 
 
 -- !query 20
 SELECT cast(null as string) > cast(1 as binary) FROM t
 -- !query 20 schema
-struct<>
+struct<(CAST(CAST(NULL AS STRING) AS BINARY) > CAST(1 AS BINARY)):boolean>
 -- !query 20 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 30
+NULL
 
 
 -- !query 21
 SELECT cast(null as string) >= cast(1 as binary) FROM t
 -- !query 21 schema
-struct<>
+struct<(CAST(CAST(NULL AS STRING) AS BINARY) >= CAST(1 AS BINARY)):boolean>
 -- !query 21 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 31
+NULL
 
 
 -- !query 22
 SELECT cast(null as string) < cast(1 as binary) FROM t
 -- !query 22 schema
-struct<>
+struct<(CAST(CAST(NULL AS STRING) AS BINARY) < CAST(1 AS BINARY)):boolean>
 -- !query 22 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 30
+NULL
 
 
 -- !query 23
 SELECT cast(null as string) <= cast(1 as binary) FROM t
 -- !query 23 schema
-struct<>
+struct<(CAST(CAST(NULL AS STRING) AS BINARY) <= CAST(1 AS BINARY)):boolean>
 -- !query 23 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 31
+NULL
 
 
 -- !query 24
 SELECT cast(null as string) <> cast(1 as binary) FROM t
 -- !query 24 schema
-struct<>
+struct<(NOT (CAST(CAST(NULL AS STRING) AS BINARY) = CAST(1 AS BINARY))):boolean>
 -- !query 24 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary;
line 1 pos 31
+NULL
 
 
 -- !query 25
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out
b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out
index 01d8393..4fa2032 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out
@@ -177,7 +177,7 @@ SELECT COUNT(*) OVER (PARTITION BY 1 ORDER BY cast('1' as binary) DESC
RANGE BET
 struct<>
 -- !query 21 output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(PARTITION BY 1 ORDER BY CAST('1' AS BINARY) DESC NULLS LAST RANGE BETWEEN
CURRENT ROW AND 1 FOLLOWING)' due to data type mismatch: The data type 'binary' used in the
order specification does not match the data type 'int' which is used in the range frame.;
line 1 pos 21
+cannot resolve 'RANGE BETWEEN CURRENT ROW AND CAST(1 AS BINARY) FOLLOWING' due to data type
mismatch: The data type of the upper bound 'binary' does not match the expected data type
'(numeric or calendarinterval)'.; line 1 pos 21
 
 
 -- !query 22
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index db9f251..050699d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -714,11 +714,11 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
 
   test("Kryo encoder: check the schema mismatch when converting DataFrame to Dataset") {
     implicit val kryoEncoder = Encoders.kryo[KryoData]
-    val df = Seq((1)).toDF("a")
+    val df = Seq((1.0)).toDF("a")
     val e = intercept[AnalysisException] {
       df.as[KryoData]
     }.message
-    assert(e.contains("cannot cast int to binary"))
+    assert(e.contains("cannot cast double to binary"))
   }
 
   test("Java encoder") {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message