carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [20/50] [abbrv] carbondata git commit: [CARBONDATA-1253] Sort_columns should not support float, double, decimal
Date Wed, 12 Jul 2017 15:04:56 GMT
[CARBONDATA-1253] Sort_columns should not support float,double,decimal

This closes #1122


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5f9741eb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5f9741eb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5f9741eb

Branch: refs/heads/datamap
Commit: 5f9741ebcb5b2b606f4d710785ce97c4d6b49229
Parents: 0d46976
Author: QiangCai <qiangcai@qq.com>
Authored: Fri Jun 30 19:51:19 2017 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Mon Jul 3 20:53:20 2017 +0800

----------------------------------------------------------------------
 ...feVariableLengthDimensionDataChunkStore.java | 12 ---
 ...afeVariableLengthDimesionDataChunkStore.java | 12 ---
 .../apache/carbondata/core/util/ByteUtil.java   | 81 --------------------
 .../carbondata/core/util/DataTypeUtil.java      | 12 ---
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 12 ++-
 5 files changed, 10 insertions(+), 119 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
index a65d745..2079811 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
@@ -25,10 +25,6 @@ import org.apache.carbondata.core.util.ByteUtil;
 
 import org.apache.spark.sql.types.BooleanType;
 import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.Decimal;
-import org.apache.spark.sql.types.DecimalType;
-import org.apache.spark.sql.types.DoubleType;
-import org.apache.spark.sql.types.FloatType;
 import org.apache.spark.sql.types.IntegerType;
 import org.apache.spark.sql.types.LongType;
 import org.apache.spark.sql.types.ShortType;
@@ -157,16 +153,8 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
         vector.putShort(vectorRow, ByteUtil.toShort(data, currentDataOffset, length));
       } else if (dt instanceof IntegerType) {
         vector.putInt(vectorRow, ByteUtil.toInt(data, currentDataOffset, length));
-      } else if (dt instanceof FloatType) {
-        vector.putFloat(vectorRow, ByteUtil.toFloat(data, currentDataOffset));
-      } else if (dt instanceof DoubleType) {
-        vector.putDouble(vectorRow, ByteUtil.toDouble(data, currentDataOffset));
       } else if (dt instanceof LongType) {
         vector.putLong(vectorRow, ByteUtil.toLong(data, currentDataOffset, length));
-      } else if (dt instanceof DecimalType) {
-        vector.putDecimal(vectorRow,
-            Decimal.apply(ByteUtil.toBigDecimal(data, currentDataOffset, length)),
-            DecimalType.MAX_PRECISION());
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
index 03ba34e..c6c98f9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
@@ -26,10 +26,6 @@ import org.apache.carbondata.core.util.ByteUtil;
 
 import org.apache.spark.sql.types.BooleanType;
 import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.Decimal;
-import org.apache.spark.sql.types.DecimalType;
-import org.apache.spark.sql.types.DoubleType;
-import org.apache.spark.sql.types.FloatType;
 import org.apache.spark.sql.types.IntegerType;
 import org.apache.spark.sql.types.LongType;
 import org.apache.spark.sql.types.ShortType;
@@ -184,16 +180,8 @@ public class UnsafeVariableLengthDimesionDataChunkStore
         vector.putShort(vectorRow, ByteUtil.toShort(value, 0, value.length));
       } else if (dt instanceof IntegerType) {
         vector.putInt(vectorRow, ByteUtil.toInt(value, 0, value.length));
-      } else if (dt instanceof FloatType) {
-        vector.putFloat(vectorRow, ByteUtil.toFloat(value, 0));
-      } else if (dt instanceof DoubleType) {
-        vector.putDouble(vectorRow, ByteUtil.toDouble(value, 0));
       } else if (dt instanceof LongType) {
         vector.putLong(vectorRow, ByteUtil.toLong(value, 0, value.length));
-      } else if (dt instanceof DecimalType) {
-        vector.putDecimal(vectorRow,
-            Decimal.apply(ByteUtil.toBigDecimal(value, 0, value.length)),
-            DecimalType.MAX_PRECISION());
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index 2761d71..b74f206 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -18,8 +18,6 @@
 package org.apache.carbondata.core.util;
 
 import java.io.UnsupportedEncodingException;
-import java.math.BigDecimal;
-import java.math.BigInteger;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 
@@ -39,8 +37,6 @@ public final class ByteUtil {
 
   public static final String UTF8_CSN = StandardCharsets.UTF_8.name();
 
-  public static final byte[] ZERO_IN_BYTES = toBytes(0);
-
   private ByteUtil() {
 
   }
@@ -531,28 +527,6 @@ public final class ByteUtil {
   }
 
   /**
-   * float => byte[]
-   *
-   * @param f
-   * @return
-   */
-  public static byte[] toBytes(final float f) {
-    // Encode it as int
-    return toBytes(Float.floatToRawIntBits(f));
-  }
-
-  /**
-   * byte[] => float
-   *
-   * @param bytes
-   * @param offset
-   * @return
-   */
-  public static float toFloat(byte[] bytes, int offset) {
-    return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
-  }
-
-  /**
    * long => byte[]
    *
    * @param val
@@ -593,61 +567,6 @@ public final class ByteUtil {
     return l ^ Long.MIN_VALUE;
   }
 
-  /**
-   * doube => byte[]
-   *
-   * @param d
-   * @return
-   */
-  public static byte[] toBytes(final double d) {
-    // Encode it as a long
-    return toBytes(Double.doubleToRawLongBits(d));
-  }
-
-  /**
-   * byte[] => double
-   *
-   * @param bytes
-   * @param offset
-   * @return
-   */
-  public static double toDouble(final byte[] bytes, final int offset) {
-    return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
-  }
-
-  /**
-   * BigDecimal => byte[]
-   *
-   * @param val
-   * @return
-   */
-  public static byte[] toBytes(BigDecimal val) {
-    byte[] valueBytes = val.unscaledValue().toByteArray();
-    byte[] result = new byte[valueBytes.length + SIZEOF_INT];
-    int offset = putInt(result, 0, val.scale());
-    putBytes(result, offset, valueBytes, 0, valueBytes.length);
-    return result;
-  }
-
-  /**
-   * byte[] => BigDecimal
-   *
-   * @param bytes
-   * @param offset
-   * @param length
-   * @return
-   */
-  public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
-    if (bytes == null || length < SIZEOF_INT + 1 || (offset + length > bytes.length))
{
-      return null;
-    }
-
-    int scale = toInt(bytes, offset, bytes.length);
-    byte[] tcBytes = new byte[length - SIZEOF_INT];
-    System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
-    return new BigDecimal(new BigInteger(tcBytes), scale);
-  }
-
   private static IllegalArgumentException explainWrongLengthOrOffset(final byte[] bytes,
       final int offset, final int length, final int expectedLength) {
     String reason;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 1b1884e..37ae5bb 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -334,14 +334,8 @@ public final class DataTypeUtil {
         return ByteUtil.toBytes(Short.parseShort(dimensionValue));
       case INT:
         return ByteUtil.toBytes(Integer.parseInt(dimensionValue));
-      case FLOAT:
-        return ByteUtil.toBytes(Float.parseFloat(dimensionValue));
       case LONG:
         return ByteUtil.toBytes(Long.parseLong(dimensionValue));
-      case DOUBLE:
-        return ByteUtil.toBytes(Double.parseDouble(dimensionValue));
-      case DECIMAL:
-        return ByteUtil.toBytes(new BigDecimal(dimensionValue));
       default:
         return ByteUtil.toBytes(dimensionValue);
     }
@@ -372,14 +366,8 @@ public final class DataTypeUtil {
           return ByteUtil.toShort(dataInBytes, 0, dataInBytes.length);
         case INT:
           return ByteUtil.toInt(dataInBytes, 0, dataInBytes.length);
-        case FLOAT:
-          return ByteUtil.toFloat(dataInBytes, 0);
         case LONG:
           return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length);
-        case DOUBLE:
-          return ByteUtil.toDouble(dataInBytes, 0);
-        case DECIMAL:
-          return ByteUtil.toBigDecimal(dataInBytes, 0, dataInBytes.length);
         default:
           return ByteUtil.toString(dataInBytes, 0, dataInBytes.length);
       }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index c565c31..8207a9d 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -551,8 +551,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser
{
         } else {
           val dataType = fields.find(x =>
             x.column.equalsIgnoreCase(column)).get.dataType.get
-          if (isComplexDimDictionaryExclude(dataType)) {
-            val errormsg = "sort_columns is unsupported for complex datatype column: " +
column
+          if (isDataTypeSupportedForSortColumn(dataType)) {
+            val errormsg = s"sort_columns is unsupported for ${dataType} datatype column:
" + column
             throw new MalformedCarbonCommandException(errormsg)
           }
         }
@@ -691,6 +691,14 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser
{
   }
 
   /**
+   * detects whether datatype is part of sort_column
+   */
+  private def isDataTypeSupportedForSortColumn(columnDataType: String): Boolean = {
+    val dataTypes = Array("array", "struct", "double", "float", "decimal")
+    dataTypes.exists(x => x.equalsIgnoreCase(columnDataType))
+  }
+
+  /**
    * detects whether datatype is part of dictionary_exclude
    */
   def isDataTypeSupportedForDictionary_Exclude(columnDataType: String): Boolean = {


Mime
View raw message