spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject spark git commit: [SPARK-16488] Fix codegen variable namespace collision in pmod and partitionBy
Date Tue, 12 Jul 2016 03:29:42 GMT
Repository: spark
Updated Branches:
  refs/heads/branch-2.0 b37177c22 -> 689261465


[SPARK-16488] Fix codegen variable namespace collision in pmod and partitionBy

This patch fixes a variable namespace collision bug in pmod and partitionBy

Regression test for one possible occurrence. A more general fix in `ExpressionEvalHelper.checkEvaluation`
will be in a subsequent PR.

Author: Sameer Agarwal <sameer@databricks.com>

Closes #14144 from sameeragarwal/codegen-bug.

(cherry picked from commit 9cc74f95edb6e4f56151966139cd0dc24e377949)
Signed-off-by: Reynold Xin <rxin@databricks.com>


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/68926146
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/68926146
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/68926146

Branch: refs/heads/branch-2.0
Commit: 689261465ad1dd443ebf764ad837243418b986ef
Parents: b37177c
Author: Sameer Agarwal <sameer@databricks.com>
Authored: Mon Jul 11 20:26:01 2016 -0700
Committer: Reynold Xin <rxin@databricks.com>
Committed: Mon Jul 11 20:29:35 2016 -0700

----------------------------------------------------------------------
 .../sql/catalyst/expressions/arithmetic.scala   | 25 ++++++++++----------
 .../sql/test/DataFrameReaderWriterSuite.scala   | 14 +++++++++++
 2 files changed, 27 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/68926146/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index 4db1352..91ffac0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -498,34 +498,35 @@ case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic
wi
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
+      val remainder = ctx.freshName("remainder")
       dataType match {
         case dt: DecimalType =>
           val decimalAdd = "$plus"
           s"""
-            ${ctx.javaType(dataType)} r = $eval1.remainder($eval2);
-            if (r.compare(new org.apache.spark.sql.types.Decimal().set(0)) < 0) {
-              ${ev.value} = (r.$decimalAdd($eval2)).remainder($eval2);
+            ${ctx.javaType(dataType)} $remainder = $eval1.remainder($eval2);
+            if ($remainder.compare(new org.apache.spark.sql.types.Decimal().set(0)) <
0) {
+              ${ev.value} = ($remainder.$decimalAdd($eval2)).remainder($eval2);
             } else {
-              ${ev.value} = r;
+              ${ev.value} = $remainder;
             }
           """
         // byte and short are casted into int when add, minus, times or divide
         case ByteType | ShortType =>
           s"""
-            ${ctx.javaType(dataType)} r = (${ctx.javaType(dataType)})($eval1 % $eval2);
-            if (r < 0) {
-              ${ev.value} = (${ctx.javaType(dataType)})((r + $eval2) % $eval2);
+            ${ctx.javaType(dataType)} $remainder = (${ctx.javaType(dataType)})($eval1 % $eval2);
+            if ($remainder < 0) {
+              ${ev.value} = (${ctx.javaType(dataType)})(($remainder + $eval2) % $eval2);
             } else {
-              ${ev.value} = r;
+              ${ev.value} = $remainder;
             }
           """
         case _ =>
           s"""
-            ${ctx.javaType(dataType)} r = $eval1 % $eval2;
-            if (r < 0) {
-              ${ev.value} = (r + $eval2) % $eval2;
+            ${ctx.javaType(dataType)} $remainder = $eval1 % $eval2;
+            if ($remainder < 0) {
+              ${ev.value} = ($remainder + $eval2) % $eval2;
             } else {
-              ${ev.value} = r;
+              ${ev.value} = $remainder;
             }
           """
       }

http://git-wip-us.apache.org/repos/asf/spark/blob/68926146/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
index 27a0a2a..75834a3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
@@ -424,6 +424,20 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSQLContext
with Be
     spark.range(10).write.orc(dir)
   }
 
+  test("pmod with partitionBy") {
+    val spark = this.spark
+    import spark.implicits._
+
+    case class Test(a: Int, b: String)
+    val data = Seq((0, "a"), (1, "b"), (1, "a"))
+    spark.createDataset(data).createOrReplaceTempView("test")
+    sql("select * from test distribute by pmod(_1, 2)")
+      .write
+      .partitionBy("_2")
+      .mode("overwrite")
+      .parquet(dir)
+  }
+
   private def testRead(
       df: => DataFrame,
       expectedResult: Seq[String],


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message