spark-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Michael Armbrust (JIRA)" <j...@apache.org>
Subject [jira] [Updated] (SPARK-9374) unix_timestamp throws AnalysisException
Date Tue, 28 Jul 2015 18:53:04 GMT

     [ https://issues.apache.org/jira/browse/SPARK-9374?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Michael Armbrust updated SPARK-9374:
------------------------------------
    Summary: unix_timestamp throws AnalysisException  (was: [Spark SQL] Throw out erorr of
"AnalysisException: nondeterministic expressions are only allowed in Project or Filter" during
the spark sql parse phase)

> unix_timestamp throws AnalysisException
> ---------------------------------------
>
>                 Key: SPARK-9374
>                 URL: https://issues.apache.org/jira/browse/SPARK-9374
>             Project: Spark
>          Issue Type: Bug
>          Components: SQL
>    Affects Versions: 1.5.0
>            Reporter: Yi Zhou
>            Priority: Blocker
>
> #################Spark SQL Query########
> INSERT INTO TABLE TEST_QUERY_0_result
> SELECT w_state, i_item_id,
>   SUM(
>     CASE WHEN (unix_timestamp(d_date,'yyyy-MM-dd') < unix_timestamp('2001-03-16','yyyy-MM-dd'))
>     THEN ws_sales_price - COALESCE(wr_refunded_cash,0)
>     ELSE 0.0 END
>   ) AS sales_before,
>   SUM(
>     CASE WHEN (unix_timestamp(d_date,'yyyy-MM-dd') >= unix_timestamp('2001-03-16','yyyy-MM-dd'))
>     THEN ws_sales_price - coalesce(wr_refunded_cash,0)
>     ELSE 0.0 END
>   ) AS sales_after
> FROM (
>   SELECT *
>   FROM web_sales ws
>   LEFT OUTER JOIN web_returns wr ON (ws.ws_order_number = wr.wr_order_number
>   AND ws.ws_item_sk = wr.wr_item_sk)
> ) a1
> JOIN item i ON a1.ws_item_sk = i.i_item_sk
> JOIN warehouse w ON a1.ws_warehouse_sk = w.w_warehouse_sk
> JOIN date_dim d ON a1.ws_sold_date_sk = d.d_date_sk
> AND unix_timestamp(d.d_date, 'yyyy-MM-dd') >= unix_timestamp('2001-03-16', 'yyyy-MM-dd')
- 30*24*60*60 --subtract 30 days in seconds
> AND unix_timestamp(d.d_date, 'yyyy-MM-dd') <= unix_timestamp('2001-03-16', 'yyyy-MM-dd')
+ 30*24*60*60 --add 30 days in seconds
> GROUP BY w_state,i_item_id
> CLUSTER BY w_state,i_item_id
> ############Error Message##################
> org.apache.spark.sql.AnalysisException: nondeterministic expressions are only allowed
in Project or Filter, found:
>  (((ws_sold_date_sk = d_date_sk) && (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date,yyyy-MM-dd)
>= (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd)
- CAST((((30 * 24) * 60) * 60), LongType)))) && (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date,yyyy-MM-dd)
<= (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd)
+ CAST((((30 * 24) * 60) * 60), LongType))))
> in operator Join Inner, Some((((ws_sold_date_sk#289L = d_date_sk#383L) && (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date#385,yyyy-MM-dd)
>= (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd)
- CAST((((30 * 24) * 60) * 60), LongType)))) && (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date#385,yyyy-MM-dd)
<= (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd)
+ CAST((((30 * 24) * 60) * 60), LongType)))))
>              ;
> 	at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.failAnalysis(CheckAnalysis.scala:37)
> 	at org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:43)
> 	at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:148)
> 	at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:49)
> 	at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:103)
> 	at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> 	at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> 	at scala.collection.immutable.List.foreach(List.scala:318)
> 	at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102)
> 	at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> 	at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> 	at scala.collection.immutable.List.foreach(List.scala:318)
> 	at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102)
> 	at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> 	at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> 	at scala.collection.immutable.List.foreach(List.scala:318)
> 	at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102)
> 	at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> 	at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> 	at scala.collection.immutable.List.foreach(List.scala:318)
> 	at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102)
> 	at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:49)
> 	at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:43)
> 	at org.apache.spark.sql.SQLContext$QueryExecution.assertAnalyzed(SQLContext.scala:976)
> 	at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:131)
> 	at org.apache.spark.sql.DataFrame$.apply(DataFrame.scala:52)
> 	at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:792)
> 	at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver.run(SparkSQLDriver.scala:61)
> 	at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processCmd(SparkSQLCLIDriver.scala:284)
> 	at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:423)
> 	at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:359)
> 	at org.apache.hadoop.hive.cli.CliDriver.processReader(CliDriver.java:456)
> 	at org.apache.hadoop.hive.cli.CliDriver.processFile(CliDriver.java:466)
> 	at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver$.main(SparkSQLCLIDriver.scala:163)
> 	at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:606)
> 	at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:666)
> 	at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:178)
> 	at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:203)
> 	at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:118)
> 	at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org


Mime
View raw message