carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [04/47] incubator-carbondata git commit: [CARBONDATA-17] - select count(*) from table where column_x = 'value' is not returning the correct count (#771)
Date Mon, 01 Aug 2016 10:05:02 GMT
[CARBONDATA-17] - select count(*) from table where column_x = 'value' is not returning the
correct count (#771)

if the first measure is having all values null.

Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/34741b48
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/34741b48
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/34741b48

Branch: refs/heads/master
Commit: 34741b48681b1a50bb46d84615dd77c17a6c44f7
Parents: 6b7b41a
Author: Mohammad Shahid Khan <mohdshahidkhan1987@gmail.com>
Authored: Fri Jul 22 15:12:17 2016 +0530
Committer: Kumar Vishal <kumarvishal.1802@gmail.com>
Committed: Fri Jul 22 15:12:17 2016 +0530

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java   |  4 ++
 .../query/aggregator/impl/CountAggregator.java  |  2 +-
 .../aggregator/impl/CountStarAggregator.java    | 51 ++++++++++++++
 .../util/MeasureAggregatorFactory.java          | 28 ++------
 .../org/apache/spark/sql/CarbonOperators.scala  |  4 +-
 .../src/test/resources/datanullmeasurecol.csv   |  3 +
 .../filterexpr/CountStarTestCase.scala          | 72 ++++++++++++++++++++
 7 files changed, 137 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/34741b48/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
index 8dbe4dd..a7a8c07 100644
--- a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
@@ -155,6 +155,10 @@ public final class CarbonCommonConstants {
    */
   public static final String COUNT = "count";
   /**
+   * Count constant
+   */
+  public static final String COUNT_STAR = "countstar";
+  /**
    * Max constant
    */
   public static final String MAX = "max";

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/34741b48/core/src/main/java/org/carbondata/query/aggregator/impl/CountAggregator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/aggregator/impl/CountAggregator.java
b/core/src/main/java/org/carbondata/query/aggregator/impl/CountAggregator.java
index 12c2061..1a040b6 100644
--- a/core/src/main/java/org/carbondata/query/aggregator/impl/CountAggregator.java
+++ b/core/src/main/java/org/carbondata/query/aggregator/impl/CountAggregator.java
@@ -42,7 +42,7 @@ public class CountAggregator implements MeasureAggregator {
   /**
    * aggregate value
    */
-  private double aggVal;
+  protected double aggVal;
 
   /**
    * Count Aggregate function which update the total count

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/34741b48/core/src/main/java/org/carbondata/query/aggregator/impl/CountStarAggregator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/aggregator/impl/CountStarAggregator.java
b/core/src/main/java/org/carbondata/query/aggregator/impl/CountStarAggregator.java
new file mode 100644
index 0000000..3b7e78c
--- /dev/null
+++ b/core/src/main/java/org/carbondata/query/aggregator/impl/CountStarAggregator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.carbondata.query.aggregator.impl;
+
+import org.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
+import org.carbondata.query.aggregator.MeasureAggregator;
+
+/**
+ * Class Description : It will return total count of values
+ */
+public class CountStarAggregator extends CountAggregator {
+
+  @Override public void agg(double newVal) {
+    super.agg(newVal);
+  }
+
+  @Override public void agg(Object newVal) {
+    aggVal++;
+  }
+
+  @Override public void agg(MeasureColumnDataChunk dataChunk, int index) {
+    aggVal++;
+  }
+
+  @Override public MeasureAggregator getCopy() {
+    CountStarAggregator aggregator = new CountStarAggregator();
+    aggregator.aggVal = aggVal;
+    return aggregator;
+  }
+
+  @Override public MeasureAggregator getNew() {
+    return new CountStarAggregator();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/34741b48/core/src/main/java/org/carbondata/query/aggregator/util/MeasureAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/aggregator/util/MeasureAggregatorFactory.java
b/core/src/main/java/org/carbondata/query/aggregator/util/MeasureAggregatorFactory.java
index a49c374..3fbf819 100644
--- a/core/src/main/java/org/carbondata/query/aggregator/util/MeasureAggregatorFactory.java
+++ b/core/src/main/java/org/carbondata/query/aggregator/util/MeasureAggregatorFactory.java
@@ -24,30 +24,7 @@ import org.carbondata.core.carbon.metadata.datatype.DataType;
 import org.carbondata.core.constants.CarbonCommonConstants;
 import org.carbondata.query.aggregator.CustomMeasureAggregator;
 import org.carbondata.query.aggregator.MeasureAggregator;
-import org.carbondata.query.aggregator.impl.AvgBigDecimalAggregator;
-import org.carbondata.query.aggregator.impl.AvgDoubleAggregator;
-import org.carbondata.query.aggregator.impl.AvgLongAggregator;
-import org.carbondata.query.aggregator.impl.AvgTimestampAggregator;
-import org.carbondata.query.aggregator.impl.CountAggregator;
-import org.carbondata.query.aggregator.impl.DistinctCountAggregatorObjectSet;
-import org.carbondata.query.aggregator.impl.DistinctCountBigDecimalAggregatorObjectSet;
-import org.carbondata.query.aggregator.impl.DistinctCountLongAggregatorObjectSet;
-import org.carbondata.query.aggregator.impl.DummyBigDecimalAggregator;
-import org.carbondata.query.aggregator.impl.DummyDoubleAggregator;
-import org.carbondata.query.aggregator.impl.DummyLongAggregator;
-import org.carbondata.query.aggregator.impl.MaxAggregator;
-import org.carbondata.query.aggregator.impl.MaxBigDecimalAggregator;
-import org.carbondata.query.aggregator.impl.MaxLongAggregator;
-import org.carbondata.query.aggregator.impl.MinAggregator;
-import org.carbondata.query.aggregator.impl.MinBigDecimalAggregator;
-import org.carbondata.query.aggregator.impl.MinLongAggregator;
-import org.carbondata.query.aggregator.impl.SumBigDecimalAggregator;
-import org.carbondata.query.aggregator.impl.SumDistinctBigDecimalAggregator;
-import org.carbondata.query.aggregator.impl.SumDistinctDoubleAggregator;
-import org.carbondata.query.aggregator.impl.SumDistinctLongAggregator;
-import org.carbondata.query.aggregator.impl.SumDoubleAggregator;
-import org.carbondata.query.aggregator.impl.SumLongAggregator;
-import org.carbondata.query.aggregator.impl.SumTimestampAggregator;
+import org.carbondata.query.aggregator.impl.*;
 import org.carbondata.query.carbon.model.CustomAggregateExpression;
 
 /**
@@ -106,6 +83,9 @@ public class MeasureAggregatorFactory {
     } else if (CarbonCommonConstants.COUNT.equalsIgnoreCase(aggregatorType)) {
       return new CountAggregator();
     }
+    else if (CarbonCommonConstants.COUNT_STAR.equalsIgnoreCase(aggregatorType)) {
+      return new CountStarAggregator();
+    }
     //
     else if (CarbonCommonConstants.MAX.equalsIgnoreCase(aggregatorType)) {
       switch (dataType) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/34741b48/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
index 9946dfd..76b7e8c 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
@@ -113,7 +113,7 @@ case class CarbonTableScan(
 
       case CountCarbon(posLiteral@PositionLiteral(Literal(star, _), _)) if star == "*" =>
         val m1 = new QueryMeasure("count(*)")
-        m1.setAggregateFunction(CarbonCommonConstants.COUNT)
+        m1.setAggregateFunction(CarbonCommonConstants.COUNT_STAR)
         m1.setQueryOrder(queryOrder)
         posLiteral.setPosition(queryOrder)
         plan.addMeasure(m1)
@@ -123,7 +123,7 @@ case class CarbonTableScan(
 
       case curr@CountCarbon(posLiteral@PositionLiteral(one, _)) =>
         val m1 = new QueryMeasure("count(*)")
-        m1.setAggregateFunction(CarbonCommonConstants.COUNT)
+        m1.setAggregateFunction(CarbonCommonConstants.COUNT_STAR)
         m1.setQueryOrder(queryOrder)
         posLiteral.setPosition(queryOrder)
         plan.addMeasure(m1)

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/34741b48/integration/spark/src/test/resources/datanullmeasurecol.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/datanullmeasurecol.csv b/integration/spark/src/test/resources/datanullmeasurecol.csv
new file mode 100644
index 0000000..40692d8
--- /dev/null
+++ b/integration/spark/src/test/resources/datanullmeasurecol.csv
@@ -0,0 +1,3 @@
+ID,date,country,name,phonetype,serialname,salary
+1,2015/7/23,china,aaa1,phone197,A234
+2,2015/7/24,china,aaa2,phone756,A453

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/34741b48/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/CountStarTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/CountStarTestCase.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/CountStarTestCase.scala
new file mode 100644
index 0000000..20881c5
--- /dev/null
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/CountStarTestCase.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.carbondata.spark.testsuite.filterexpr
+
+import java.io.File
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+import org.carbondata.core.constants.CarbonCommonConstants
+import org.carbondata.core.util.CarbonProperties
+import org.scalatest.BeforeAndAfterAll
+
+/**
+  * Test Class for filter expression query on String datatypes
+  *
+  * @author N00902756
+  *
+  */
+class CountStarTestCase extends QueryTest with BeforeAndAfterAll {
+
+  override def beforeAll {
+    sql("drop table if exists filtertestTables")
+
+
+    sql("CREATE TABLE filterTimestampDataType (ID int, date Timestamp, country String, "
+
+      "name String, phonetype String, serialname String, salary int) " +
+      "STORED BY 'org.apache.carbondata.format'"
+    )
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/mm/dd")
+    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+      .getCanonicalPath
+    var csvFilePath = currentDirectory + "/src/test/resources/datanullmeasurecol.csv";
+      sql(
+      s"LOAD DATA LOCAL INPATH '" + csvFilePath + "' INTO TABLE " +
+        s"filterTimestampDataType " +
+        s"OPTIONS('DELIMITER'= ',', " +
+        s"'FILEHEADER'= '')"
+    )
+  }
+
+  test("select count ") {
+    checkAnswer(
+      sql("select count(*) from filterTimestampDataType where country='china'"),
+      Seq(Row(2))
+    )
+  }
+
+  override def afterAll {
+    sql("drop table noloadtable")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+  }
+}
\ No newline at end of file


Mime
View raw message