From commits-return-36321-archive-asf-public=cust-asf.ponee.io@spark.apache.org Fri Mar 22 17:10:31 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 6D6AB180657 for ; Fri, 22 Mar 2019 18:10:30 +0100 (CET) Received: (qmail 82706 invoked by uid 500); 22 Mar 2019 17:10:29 -0000 Mailing-List: contact commits-help@spark.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list commits@spark.apache.org Received: (qmail 82691 invoked by uid 99); 22 Mar 2019 17:10:29 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 22 Mar 2019 17:10:29 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id EA78A850AF; Fri, 22 Mar 2019 17:10:28 +0000 (UTC) Date: Fri, 22 Mar 2019 17:10:28 +0000 To: "commits@spark.apache.org" Subject: [spark] branch master updated: [SPARK-27174][SQL] Add support for casting integer types to binary MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <155327462686.16346.13843807171213461886@gitbox.apache.org> From: wenchen@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: spark X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 8204dc1e548b87aabaf36c5800592bafd44e4419 X-Git-Newrev: 8efc5ec72e2f5899547941010e22c023d6cb86b3 X-Git-Rev: 8efc5ec72e2f5899547941010e22c023d6cb86b3 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. wenchen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/master by this push: new 8efc5ec [SPARK-27174][SQL] Add support for casting integer types to binary 8efc5ec is described below commit 8efc5ec72e2f5899547941010e22c023d6cb86b3 Author: Martin Junghanns AuthorDate: Fri Mar 22 10:09:35 2019 -0700 [SPARK-27174][SQL] Add support for casting integer types to binary Co-authored-by: Philip Stutz ## What changes were proposed in this pull request? This PR adds support for casting * `ByteType` * `ShortType` * `IntegerType` * `LongType` to `BinaryType`. ## How was this patch tested? We added unit tests for casting instances of the above types. For validation, we used Javas `DataOutputStream` to compare the resulting byte array with the result of `Cast`. We state that the contribution is our original work and that we license the work to the project under the project’s open source license. cloud-fan we'd appreciate a review if you find the time, thx Closes #24107 from s1ck/cast_to_binary. Authored-by: Martin Junghanns Signed-off-by: Wenchen Fan --- .../spark/sql/catalyst/expressions/Cast.scala | 11 +- .../spark/sql/catalyst/util/NumberConverter.scala | 35 ++++++ .../sql/catalyst/analysis/AnalysisErrorSuite.scala | 5 - .../sql/catalyst/util/NumberConverterSuite.scala | 48 ++++++++- .../src/test/resources/sql-tests/inputs/cast.sql | 13 +++ .../test/resources/sql-tests/results/cast.sql.out | 84 +++++++++++++-- .../typeCoercion/native/binaryComparison.sql.out | 120 +++++++++------------ .../native/windowFrameCoercion.sql.out | 2 +- .../scala/org/apache/spark/sql/DatasetSuite.scala | 4 +- 9 files changed, 234 insertions(+), 88 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 72cb6b2..848195f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -44,6 +44,7 @@ object Cast { case (_, StringType) => true case (StringType, BinaryType) => true + case (_: IntegralType, BinaryType) => true case (StringType, BooleanType) => true case (DateType, BooleanType) => true @@ -326,6 +327,10 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String // BinaryConverter private[this] def castToBinary(from: DataType): Any => Any = from match { case StringType => buildCast[UTF8String](_, _.getBytes) + case ByteType => buildCast[Byte](_, NumberConverter.toBinary) + case ShortType => buildCast[Short](_, NumberConverter.toBinary) + case IntegerType => buildCast[Int](_, NumberConverter.toBinary) + case LongType => buildCast[Long](_, NumberConverter.toBinary) } // UDFToBoolean @@ -908,7 +913,11 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String private[this] def castToBinaryCode(from: DataType): CastFunction = from match { case StringType => - (c, evPrim, evNull) => code"$evPrim = $c.getBytes();" + (c, evPrim, evNull) => + code"$evPrim = $c.getBytes();" + case _: IntegralType => + (c, evPrim, evNull) => + code"$evPrim = ${NumberConverter.getClass.getName.stripSuffix("$")}.toBinary($c);" } private[this] def castToDateCode( diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala index 9c3f6b7..7dbdd1e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala @@ -169,4 +169,39 @@ object NumberConverter { } UTF8String.fromBytes(java.util.Arrays.copyOfRange(temp, resultStartPos, temp.length)) } + + def toBinary(l: Long): Array[Byte] = { + val result = new Array[Byte](8) + result(0) = (l >>> 56 & 0xFF).toByte + result(1) = (l >>> 48 & 0xFF).toByte + result(2) = (l >>> 40 & 0xFF).toByte + result(3) = (l >>> 32 & 0xFF).toByte + result(4) = (l >>> 24 & 0xFF).toByte + result(5) = (l >>> 16 & 0xFF).toByte + result(6) = (l >>> 8 & 0xFF).toByte + result(7) = (l & 0xFF).toByte + result + } + + def toBinary(i: Int): Array[Byte] = { + val result = new Array[Byte](4) + result(0) = (i >>> 24 & 0xFF).toByte + result(1) = (i >>> 16 & 0xFF).toByte + result(2) = (i >>> 8 & 0xFF).toByte + result(3) = (i & 0xFF).toByte + result + } + + def toBinary(s: Short): Array[Byte] = { + val result = new Array[Byte](2) + result(0) = (s >>> 8 & 0xFF).toByte + result(1) = (s & 0xFF).toByte + result + } + + def toBinary(s: Byte): Array[Byte] = { + val result = new Array[Byte](1) + result(0) = s + result + } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala index 129ce3b..ec71c1c 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala @@ -216,11 +216,6 @@ class AnalysisErrorSuite extends AnalysisTest { "Invalid usage of '*'" :: "in expression 'sum'" :: Nil) errorTest( - "bad casts", - testRelation.select(Literal(1).cast(BinaryType).as('badCast)), - "cannot cast" :: Literal(1).dataType.simpleString :: BinaryType.simpleString :: Nil) - - errorTest( "sorting by unsupported column types", mapRelation.orderBy('map.asc), "sort" :: "type" :: "map" :: Nil) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala index 13265a1..ec73f45 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala @@ -17,8 +17,11 @@ package org.apache.spark.sql.catalyst.util +import java.nio.ByteBuffer +import java.nio.ByteOrder.BIG_ENDIAN + import org.apache.spark.SparkFunSuite -import org.apache.spark.sql.catalyst.util.NumberConverter.convert +import org.apache.spark.sql.catalyst.util.NumberConverter.{convert, toBinary} import org.apache.spark.unsafe.types.UTF8String class NumberConverterSuite extends SparkFunSuite { @@ -37,4 +40,47 @@ class NumberConverterSuite extends SparkFunSuite { checkConv("11abc", 10, 16, "B") } + test("byte to binary") { + checkToBinary(0.toByte) + checkToBinary(1.toByte) + checkToBinary(-1.toByte) + checkToBinary(Byte.MaxValue) + checkToBinary(Byte.MinValue) + } + + test("short to binary") { + checkToBinary(0.toShort) + checkToBinary(1.toShort) + checkToBinary(-1.toShort) + checkToBinary(Short.MaxValue) + checkToBinary(Short.MinValue) + } + + test("integer to binary") { + checkToBinary(0) + checkToBinary(1) + checkToBinary(-1) + checkToBinary(Int.MaxValue) + checkToBinary(Int.MinValue) + } + + test("long to binary") { + checkToBinary(0L) + checkToBinary(1L) + checkToBinary(-1L) + checkToBinary(Long.MaxValue) + checkToBinary(Long.MinValue) + } + + def checkToBinary[T](in: T): Unit = in match { + case b: Byte => + assert(toBinary(b) === ByteBuffer.allocate(1).order(BIG_ENDIAN).put(b).array()) + case s: Short => + assert(toBinary(s) === ByteBuffer.allocate(2).order(BIG_ENDIAN).putShort(s).array()) + case i: Int => + assert(toBinary(i) === ByteBuffer.allocate(4).order(BIG_ENDIAN).putInt(i).array()) + case l: Long => + assert(toBinary(l) === ByteBuffer.allocate(8).order(BIG_ENDIAN).putLong(l).array()) + } + } diff --git a/sql/core/src/test/resources/sql-tests/inputs/cast.sql b/sql/core/src/test/resources/sql-tests/inputs/cast.sql index 629df59..7244cd3 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/cast.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/cast.sql @@ -40,6 +40,19 @@ SELECT CAST('-9223372036854775809' AS long); SELECT CAST('9223372036854775807' AS long); SELECT CAST('9223372036854775808' AS long); +-- cast string to its binary representation +SELECT HEX(CAST('abc' AS binary)); + +-- cast integral values to their corresponding binary representation +SELECT HEX(CAST(CAST(123 AS byte) AS binary)); +SELECT HEX(CAST(CAST(-123 AS byte) AS binary)); +SELECT HEX(CAST(123S AS binary)); +SELECT HEX(CAST(-123S AS binary)); +SELECT HEX(CAST(123 AS binary)); +SELECT HEX(CAST(-123 AS binary)); +SELECT HEX(CAST(123L AS binary)); +SELECT HEX(CAST(-123L AS binary)); + DESC FUNCTION boolean; DESC FUNCTION EXTENDED boolean; -- TODO: migrate all cast tests here. diff --git a/sql/core/src/test/resources/sql-tests/results/cast.sql.out b/sql/core/src/test/resources/sql-tests/results/cast.sql.out index 9c5f455..21b18e9 100644 --- a/sql/core/src/test/resources/sql-tests/results/cast.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/cast.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 24 +-- Number of queries: 33 -- !query 0 @@ -179,20 +179,92 @@ NULL -- !query 22 -DESC FUNCTION boolean +SELECT HEX(CAST('abc' AS binary)) -- !query 22 schema -struct +struct -- !query 22 output +616263 + + +-- !query 23 +SELECT HEX(CAST(CAST(123 AS byte) AS binary)) +-- !query 23 schema +struct +-- !query 23 output +7B + + +-- !query 24 +SELECT HEX(CAST(CAST(-123 AS byte) AS binary)) +-- !query 24 schema +struct +-- !query 24 output +85 + + +-- !query 25 +SELECT HEX(CAST(123S AS binary)) +-- !query 25 schema +struct +-- !query 25 output +007B + + +-- !query 26 +SELECT HEX(CAST(-123S AS binary)) +-- !query 26 schema +struct +-- !query 26 output +FF85 + + +-- !query 27 +SELECT HEX(CAST(123 AS binary)) +-- !query 27 schema +struct +-- !query 27 output +0000007B + + +-- !query 28 +SELECT HEX(CAST(-123 AS binary)) +-- !query 28 schema +struct +-- !query 28 output +FFFFFF85 + + +-- !query 29 +SELECT HEX(CAST(123L AS binary)) +-- !query 29 schema +struct +-- !query 29 output +000000000000007B + + +-- !query 30 +SELECT HEX(CAST(-123L AS binary)) +-- !query 30 schema +struct +-- !query 30 output +FFFFFFFFFFFFFF85 + + +-- !query 31 +DESC FUNCTION boolean +-- !query 31 schema +struct +-- !query 31 output Class: org.apache.spark.sql.catalyst.expressions.Cast Function: boolean Usage: boolean(expr) - Casts the value `expr` to the target data type `boolean`. --- !query 23 +-- !query 32 DESC FUNCTION EXTENDED boolean --- !query 23 schema +-- !query 32 schema struct --- !query 23 output +-- !query 32 output Class: org.apache.spark.sql.catalyst.expressions.Cast Extended Usage: No example/argument for boolean. diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out index 2914d60..b764ce4 100644 --- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out @@ -13,217 +13,193 @@ struct<> -- !query 1 SELECT cast(1 as binary) = '1' FROM t -- !query 1 schema -struct<> +struct<(CAST(1 AS BINARY) = CAST(1 AS BINARY)):boolean> -- !query 1 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7 +false -- !query 2 SELECT cast(1 as binary) > '2' FROM t -- !query 2 schema -struct<> +struct<(CAST(1 AS BINARY) > CAST(2 AS BINARY)):boolean> -- !query 2 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7 +false -- !query 3 SELECT cast(1 as binary) >= '2' FROM t -- !query 3 schema -struct<> +struct<(CAST(1 AS BINARY) >= CAST(2 AS BINARY)):boolean> -- !query 3 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7 +false -- !query 4 SELECT cast(1 as binary) < '2' FROM t -- !query 4 schema -struct<> +struct<(CAST(1 AS BINARY) < CAST(2 AS BINARY)):boolean> -- !query 4 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7 +true -- !query 5 SELECT cast(1 as binary) <= '2' FROM t -- !query 5 schema -struct<> +struct<(CAST(1 AS BINARY) <= CAST(2 AS BINARY)):boolean> -- !query 5 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7 +true -- !query 6 SELECT cast(1 as binary) <> '2' FROM t -- !query 6 schema -struct<> +struct<(NOT (CAST(1 AS BINARY) = CAST(2 AS BINARY))):boolean> -- !query 6 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7 +true -- !query 7 SELECT cast(1 as binary) = cast(null as string) FROM t -- !query 7 schema -struct<> +struct<(CAST(1 AS BINARY) = CAST(CAST(NULL AS STRING) AS BINARY)):boolean> -- !query 7 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7 +NULL -- !query 8 SELECT cast(1 as binary) > cast(null as string) FROM t -- !query 8 schema -struct<> +struct<(CAST(1 AS BINARY) > CAST(CAST(NULL AS STRING) AS BINARY)):boolean> -- !query 8 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7 +NULL -- !query 9 SELECT cast(1 as binary) >= cast(null as string) FROM t -- !query 9 schema -struct<> +struct<(CAST(1 AS BINARY) >= CAST(CAST(NULL AS STRING) AS BINARY)):boolean> -- !query 9 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7 +NULL -- !query 10 SELECT cast(1 as binary) < cast(null as string) FROM t -- !query 10 schema -struct<> +struct<(CAST(1 AS BINARY) < CAST(CAST(NULL AS STRING) AS BINARY)):boolean> -- !query 10 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7 +NULL -- !query 11 SELECT cast(1 as binary) <= cast(null as string) FROM t -- !query 11 schema -struct<> +struct<(CAST(1 AS BINARY) <= CAST(CAST(NULL AS STRING) AS BINARY)):boolean> -- !query 11 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7 +NULL -- !query 12 SELECT cast(1 as binary) <> cast(null as string) FROM t -- !query 12 schema -struct<> +struct<(NOT (CAST(1 AS BINARY) = CAST(CAST(NULL AS STRING) AS BINARY))):boolean> -- !query 12 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7 +NULL -- !query 13 SELECT '1' = cast(1 as binary) FROM t -- !query 13 schema -struct<> +struct<(CAST(1 AS BINARY) = CAST(1 AS BINARY)):boolean> -- !query 13 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 13 +false -- !query 14 SELECT '2' > cast(1 as binary) FROM t -- !query 14 schema -struct<> +struct<(CAST(2 AS BINARY) > CAST(1 AS BINARY)):boolean> -- !query 14 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 13 +true -- !query 15 SELECT '2' >= cast(1 as binary) FROM t -- !query 15 schema -struct<> +struct<(CAST(2 AS BINARY) >= CAST(1 AS BINARY)):boolean> -- !query 15 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 14 +true -- !query 16 SELECT '2' < cast(1 as binary) FROM t -- !query 16 schema -struct<> +struct<(CAST(2 AS BINARY) < CAST(1 AS BINARY)):boolean> -- !query 16 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 13 +false -- !query 17 SELECT '2' <= cast(1 as binary) FROM t -- !query 17 schema -struct<> +struct<(CAST(2 AS BINARY) <= CAST(1 AS BINARY)):boolean> -- !query 17 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 14 +false -- !query 18 SELECT '2' <> cast(1 as binary) FROM t -- !query 18 schema -struct<> +struct<(NOT (CAST(2 AS BINARY) = CAST(1 AS BINARY))):boolean> -- !query 18 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 14 +true -- !query 19 SELECT cast(null as string) = cast(1 as binary) FROM t -- !query 19 schema -struct<> +struct<(CAST(CAST(NULL AS STRING) AS BINARY) = CAST(1 AS BINARY)):boolean> -- !query 19 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 30 +NULL -- !query 20 SELECT cast(null as string) > cast(1 as binary) FROM t -- !query 20 schema -struct<> +struct<(CAST(CAST(NULL AS STRING) AS BINARY) > CAST(1 AS BINARY)):boolean> -- !query 20 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 30 +NULL -- !query 21 SELECT cast(null as string) >= cast(1 as binary) FROM t -- !query 21 schema -struct<> +struct<(CAST(CAST(NULL AS STRING) AS BINARY) >= CAST(1 AS BINARY)):boolean> -- !query 21 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 31 +NULL -- !query 22 SELECT cast(null as string) < cast(1 as binary) FROM t -- !query 22 schema -struct<> +struct<(CAST(CAST(NULL AS STRING) AS BINARY) < CAST(1 AS BINARY)):boolean> -- !query 22 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 30 +NULL -- !query 23 SELECT cast(null as string) <= cast(1 as binary) FROM t -- !query 23 schema -struct<> +struct<(CAST(CAST(NULL AS STRING) AS BINARY) <= CAST(1 AS BINARY)):boolean> -- !query 23 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 31 +NULL -- !query 24 SELECT cast(null as string) <> cast(1 as binary) FROM t -- !query 24 schema -struct<> +struct<(NOT (CAST(CAST(NULL AS STRING) AS BINARY) = CAST(1 AS BINARY))):boolean> -- !query 24 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 31 +NULL -- !query 25 diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out index 01d8393..4fa2032 100644 --- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out @@ -177,7 +177,7 @@ SELECT COUNT(*) OVER (PARTITION BY 1 ORDER BY cast('1' as binary) DESC RANGE BET struct<> -- !query 21 output org.apache.spark.sql.AnalysisException -cannot resolve '(PARTITION BY 1 ORDER BY CAST('1' AS BINARY) DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING)' due to data type mismatch: The data type 'binary' used in the order specification does not match the data type 'int' which is used in the range frame.; line 1 pos 21 +cannot resolve 'RANGE BETWEEN CURRENT ROW AND CAST(1 AS BINARY) FOLLOWING' due to data type mismatch: The data type of the upper bound 'binary' does not match the expected data type '(numeric or calendarinterval)'.; line 1 pos 21 -- !query 22 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala index db9f251..050699d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala @@ -714,11 +714,11 @@ class DatasetSuite extends QueryTest with SharedSQLContext { test("Kryo encoder: check the schema mismatch when converting DataFrame to Dataset") { implicit val kryoEncoder = Encoders.kryo[KryoData] - val df = Seq((1)).toDF("a") + val df = Seq((1.0)).toDF("a") val e = intercept[AnalysisException] { df.as[KryoData] }.message - assert(e.contains("cannot cast int to binary")) + assert(e.contains("cannot cast double to binary")) } test("Java encoder") { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org For additional commands, e-mail: commits-help@spark.apache.org