carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [22/56] [abbrv] carbondata git commit: partitioned by all primitive data type
Date Tue, 20 Jun 2017 07:29:28 GMT
partitioned by all primitive data type


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/de5346f3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/de5346f3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/de5346f3

Branch: refs/heads/streaming_ingest
Commit: de5346f3cd34f69ce25234ee3fb1cc146c3dc062
Parents: 447049a
Author: QiangCai <david.caiq@gmail.com>
Authored: Fri Jun 9 19:01:34 2017 +0800
Committer: Venkata Ramana G <ramana.gollamudi@huawei.com>
Committed: Thu Jun 15 13:47:25 2017 +0530

----------------------------------------------------------------------
 .../filter/partition/EqualToFilterImpl.java     |    5 +
 .../scan/filter/partition/InFilterImpl.java     |    5 +
 .../scan/filter/partition/RangeFilterImpl.java  |    4 +
 .../test/resources/alldatatypeforpartition.csv  |    4 +
 .../TestAllDataTypeForPartitionTable.scala      | 1240 ++++++++++++++++++
 .../spark/util/DataTypeConverterUtil.scala      |    5 +
 .../spark/rdd/CarbonDataRDDFactory.scala        |    2 +-
 .../spark/rdd/CarbonDataRDDFactory.scala        |    2 +-
 8 files changed, 1265 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/de5346f3/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/EqualToFilterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/EqualToFilterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/EqualToFilterImpl.java
index 8542a75..4df892d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/EqualToFilterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/EqualToFilterImpl.java
@@ -20,10 +20,12 @@ package org.apache.carbondata.core.scan.filter.partition;
 import java.util.BitSet;
 
 import org.apache.carbondata.core.metadata.schema.PartitionInfo;
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType;
 import org.apache.carbondata.core.scan.expression.LiteralExpression;
 import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
 import org.apache.carbondata.core.scan.partition.PartitionUtil;
 import org.apache.carbondata.core.scan.partition.Partitioner;
+import org.apache.carbondata.core.util.ByteUtil;
 
 /**
  * the implement of EqualTo filter
@@ -47,6 +49,9 @@ public class EqualToFilterImpl implements PartitionFilterIntf {
       Object value = PartitionUtil.getDataBasedOnDataTypeForFilter(
           literal.getLiteralExpValue().toString(),
           partitionInfo.getColumnSchemaList().get(0).getDataType());
+      if (PartitionType.RANGE == partitionInfo.getPartitionType() && value instanceof
String) {
+        value = ByteUtil.toBytes((String)value);
+      }
       partitionMap.set(partitioner.getPartition(value));
     }
     return partitionMap;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/de5346f3/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/InFilterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/InFilterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/InFilterImpl.java
index 18bec64..0f45b32 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/InFilterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/InFilterImpl.java
@@ -20,12 +20,14 @@ package org.apache.carbondata.core.scan.filter.partition;
 import java.util.BitSet;
 
 import org.apache.carbondata.core.metadata.schema.PartitionInfo;
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType;
 import org.apache.carbondata.core.scan.expression.Expression;
 import org.apache.carbondata.core.scan.expression.LiteralExpression;
 import org.apache.carbondata.core.scan.expression.conditional.InExpression;
 import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
 import org.apache.carbondata.core.scan.partition.PartitionUtil;
 import org.apache.carbondata.core.scan.partition.Partitioner;
+import org.apache.carbondata.core.util.ByteUtil;
 
 /**
  * the implement of In filter
@@ -48,6 +50,9 @@ public class InFilterImpl implements PartitionFilterIntf {
       Object value = PartitionUtil.getDataBasedOnDataTypeForFilter(
           literal.getLiteralExpValue().toString(),
           partitionInfo.getColumnSchemaList().get(0).getDataType());
+      if (PartitionType.RANGE == partitionInfo.getPartitionType() && value instanceof
String) {
+        value = ByteUtil.toBytes((String)value);
+      }
       partitionMap.set(partitioner.getPartition(value));
     }
     return partitionMap;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/de5346f3/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/RangeFilterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/RangeFilterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/RangeFilterImpl.java
index b8a7640..697be32 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/RangeFilterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/RangeFilterImpl.java
@@ -25,6 +25,7 @@ import org.apache.carbondata.core.scan.partition.ListPartitioner;
 import org.apache.carbondata.core.scan.partition.PartitionUtil;
 import org.apache.carbondata.core.scan.partition.Partitioner;
 import org.apache.carbondata.core.scan.partition.RangePartitioner;
+import org.apache.carbondata.core.util.ByteUtil;
 
 /**
  * the implement of Range filter(include <=, <, >=, >)
@@ -57,6 +58,9 @@ public class RangeFilterImpl implements PartitionFilterIntf {
         Object filterValueOfRange = PartitionUtil.getDataBasedOnDataTypeForFilter(
             literal.getLiteralExpValue().toString(),
             partitionInfo.getColumnSchemaList().get(0).getDataType());
+        if (filterValueOfRange instanceof String) {
+          filterValueOfRange = ByteUtil.toBytes((String)filterValueOfRange);
+        }
         return PartitionFilterUtil.getPartitionMapForRangeFilter(partitionInfo,
             (RangePartitioner) partitioner, filterValueOfRange, isGreaterThan, isEqualTo);
       default:

http://git-wip-us.apache.org/repos/asf/carbondata/blob/de5346f3/integration/spark-common-test/src/test/resources/alldatatypeforpartition.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/alldatatypeforpartition.csv
b/integration/spark-common-test/src/test/resources/alldatatypeforpartition.csv
new file mode 100644
index 0000000..a8e9cb2
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/alldatatypeforpartition.csv
@@ -0,0 +1,4 @@
+smallIntField,intField,bigIntField,floatField,doubleField,decimalField,timestampField,dateField,stringField,varcharField,charField,arrayField,structField
+-32768,-2147483648,-9223372036854775808,-2147483648.1,-9223372036854775808.1,-9223372036854775808.1234,2017-06-11
00:00:01,2017-06-11,abc1,abcd1,abcde1,a$b$c$1,a$b$1
+128,32768,2147483648,2147483647.1,9223372036854775807.1,9223372036854775807.1234,2017-06-12
23:59:59,2017-06-12,abc2,abcd2,abcde2,a$b$c$2,a$b$2
+32767,2147483647,9223372036854775807,2147483648.1,9223372036854775808.1,9223372036854775808.1234,2017-06-13
23:59:59,2017-06-13,abc3,abcd3,abcde3,a$b$c$3,a$b$3
\ No newline at end of file


Mime
View raw message