spark-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "someshwar kale (JIRA)" <j...@apache.org>
Subject [jira] [Created] (SPARK-20698) =, ==, > is not working as expected when used in sql query
Date Wed, 10 May 2017 19:12:04 GMT
someshwar kale created SPARK-20698:
--------------------------------------

             Summary: =, ==, > is not working as expected when used in sql query
                 Key: SPARK-20698
                 URL: https://issues.apache.org/jira/browse/SPARK-20698
             Project: Spark
          Issue Type: Bug
          Components: SQL
    Affects Versions: 1.6.2
         Environment: windows
            Reporter: someshwar kale
            Priority: Critical
             Fix For: 1.6.2


I have written below spark program- its not working as expected

++++++++++++++++++++++++
package computedBatch;

import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.hive.HiveContext;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;


public class ArithmeticIssueTest {
    private transient JavaSparkContext javaSparkContext;
    private transient SQLContext sqlContext;

    public ArithmeticIssueTest() {
        Logger.getLogger("org").setLevel(Level.OFF);
        Logger.getLogger("akka").setLevel(Level.OFF);
        SparkConf conf = new SparkConf().setAppName("ArithmeticIssueTest").setMaster("local[4]");
        javaSparkContext = new JavaSparkContext(conf);
        sqlContext = new HiveContext(javaSparkContext);
    }

    public static void main(String[] args) {
      ArithmeticIssueTest arithmeticIssueTest = new ArithmeticIssueTest();
        arithmeticIssueTest.execute();
    }

    private void execute(){
        List<String> data = Arrays.asList(
                "a1,1494389759,99.8793003568,325.389705932",
                "a1,1494389759,99.9472573803,325.27559502",
                "a1,1494389759,99.7887233987,325.334374851",
                "a1,1494389759,99.9547800925,325.371537062",
                "a1,1494389759,99.8039111691,325.305285877",
                "a1,1494389759,99.8342317379,325.24881354",
                "a1,1494389759,99.9849449235,325.396678931",
                "a1,1494389759,99.9396731311,325.336115345",
                "a1,1494389759,99.9320915068,325.242622938",
                "a1,1494389759,99.8943333669,325.320965146",
                "a1,1494389759,99.7735359781,325.345168334",
                "a1,1494389759,99.9698837734,325.352291407",
                "a1,1494389759,99.8418330703,325.296539372",
                "a1,1494389759,99.796315751,325.347570632",
                "a1,1494389759,99.7811931613,325.351137315",
                "a1,1494389759,99.9773765104,325.218131741",
                "a1,1494389759,99.8189825201,325.288197381",
                "a1,1494389759,99.8115005369,325.282327633",
                "a1,1494389759,99.9924539722,325.24048614",
                "a1,1494389759,99.9170191204,325.299431664");
        JavaRDD<String> rawData = javaSparkContext.parallelize(data);
        List<StructField> fields = new ArrayList<>();
        fields.add(DataTypes.createStructField("ASSET_ID", DataTypes.StringType, true));
        fields.add(DataTypes.createStructField("TIMESTAMP", DataTypes.LongType, true));
        fields.add(DataTypes.createStructField("fuel", DataTypes.DoubleType, true));
        fields.add(DataTypes.createStructField("temperature", DataTypes.DoubleType, true));
        StructType schema = DataTypes.createStructType(fields);
        JavaRDD<Row> rowRDD = rawData.map(
                (Function<String, Row>) record -> {
                    String[] fields1 = record.split(",");
                    return RowFactory.create(
                            fields1[0].trim(),
                            Long.parseLong(fields1[1].trim()),
                            Double.parseDouble(fields1[2].trim()),
                            Double.parseDouble(fields1[3].trim()));
                });
        DataFrame df = sqlContext.createDataFrame(rowRDD, schema);
        df.show(false);
        df.registerTempTable("x_linkx1087571272_filtered");

        sqlContext.sql("SELECT x_linkx1087571272_filtered.ASSET_ID, count(case when x_linkx1087571272_filtered"
+
                ".temperature=325.0 then 1 else 0 end) AS xsumptionx1582594572, max(x_linkx1087571272_filtered"
+
                ".TIMESTAMP) AS eventTime  FROM x_linkx1087571272_filtered GROUP BY x_linkx1087571272_filtered"
+
                ".ASSET_ID").show(false);

        sqlContext.sql("SELECT x_linkx1087571272_filtered.ASSET_ID, count(case when x_linkx1087571272_filtered"
+
                ".fuel>99.8 then 1 else 0 end) AS xnsumptionx352569416, max(x_linkx1087571272_filtered.TIMESTAMP)
AS " +
                "eventTime  FROM x_linkx1087571272_filtered GROUP BY x_linkx1087571272_filtered.ASSET_ID").show(false);

//        +++++++++
        sqlContext.sql("SELECT x_linkx1087571272_filtered.ASSET_ID, count(case when x_linkx1087571272_filtered"
+
                ".temperature==325.0 then 1 else 0 end) AS xsumptionx1582594572, max(x_linkx1087571272_filtered"
+
                ".TIMESTAMP) AS eventTime  FROM x_linkx1087571272_filtered GROUP BY x_linkx1087571272_filtered"
+
                ".ASSET_ID").show(false);
    }
}

++++++++++++++++++++++++++++++

Logs-

+--------+----------+-------------+-------------+
|ASSET_ID|TIMESTAMP |fuel         |temperature  |
+--------+----------+-------------+-------------+
|a1      |1494389759|99.8793003568|325.389705932|
|a1      |1494389759|99.9472573803|325.27559502 |
|a1      |1494389759|99.7887233987|325.334374851|
|a1      |1494389759|99.9547800925|325.371537062|
|a1      |1494389759|99.8039111691|325.305285877|
|a1      |1494389759|99.8342317379|325.24881354 |
|a1      |1494389759|99.9849449235|325.396678931|
|a1      |1494389759|99.9396731311|325.336115345|
|a1      |1494389759|99.9320915068|325.242622938|
|a1      |1494389759|99.8943333669|325.320965146|
|a1      |1494389759|99.7735359781|325.345168334|
|a1      |1494389759|99.9698837734|325.352291407|
|a1      |1494389759|99.8418330703|325.296539372|
|a1      |1494389759|99.796315751 |325.347570632|
|a1      |1494389759|99.7811931613|325.351137315|
|a1      |1494389759|99.9773765104|325.218131741|
|a1      |1494389759|99.8189825201|325.288197381|
|a1      |1494389759|99.8115005369|325.282327633|
|a1      |1494389759|99.9924539722|325.24048614 |
|a1      |1494389759|99.9170191204|325.299431664|
+--------+----------+-------------+-------------+

17/05/11 00:22:08 INFO ParseDriver: Parsing command: SELECT x_linkx1087571272_filtered.ASSET_ID,
count(case when x_linkx1087571272_filtered.temperature=325.0 then 1 else 0 end) AS xsumptionx1582594572,
max(x_linkx1087571272_filtered.TIMESTAMP) AS eventTime  FROM x_linkx1087571272_filtered GROUP
BY x_linkx1087571272_filtered.ASSET_ID
17/05/11 00:22:09 INFO ParseDriver: Parse Completed
[Stage 5:======================================================>(198 + 1) / 199]+--------+--------------------+----------+
|ASSET_ID|xsumptionx1582594572|eventTime |
+--------+--------------------+----------+
|a1      |20                  |1494389759|
+--------+--------------------+----------+

17/05/11 00:22:16 INFO ParseDriver: Parsing command: SELECT x_linkx1087571272_filtered.ASSET_ID,
count(case when x_linkx1087571272_filtered.fuel>99.8 then 1 else 0 end) AS xnsumptionx352569416,
max(x_linkx1087571272_filtered.TIMESTAMP) AS eventTime  FROM x_linkx1087571272_filtered GROUP
BY x_linkx1087571272_filtered.ASSET_ID
17/05/11 00:22:16 INFO ParseDriver: Parse Completed
+--------+--------------------+----------+
|ASSET_ID|xnsumptionx352569416|eventTime |
+--------+--------------------+----------+
|a1      |20                  |1494389759|
+--------+--------------------+----------+

17/05/11 00:22:24 INFO ParseDriver: Parsing command: SELECT x_linkx1087571272_filtered.ASSET_ID,
count(case when x_linkx1087571272_filtered.temperature==325.0 then 1 else 0 end) AS xsumptionx1582594572,
max(x_linkx1087571272_filtered.TIMESTAMP) AS eventTime  FROM x_linkx1087571272_filtered GROUP
BY x_linkx1087571272_filtered.ASSET_ID
17/05/11 00:22:24 INFO ParseDriver: Parse Completed
[Stage 13:==========================================>           (158 + 4) / 199]+--------+--------------------+----------+
|ASSET_ID|xsumptionx1582594572|eventTime |
+--------+--------------------+----------+
|a1      |20                  |1494389759|
+--------+--------------------+----------+

both the queries are resulting to wrong values




--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org


Mime
View raw message