parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject [09/50] [abbrv] parquet-mr git commit: PARQUET-484: Warn when Decimal is stored as INT64 while could be stored as INT32
Date Thu, 19 Jan 2017 01:27:20 GMT
PARQUET-484: Warn when Decimal is stored as INT64 while could be stored as INT32

Below is documented in [LogicalTypes.md](https://github.com/Parquet/parquet-format/blob/master/LogicalTypes.md#decimal):

> int32: for 1 <= precision <= 9
> int64: for 1 <= precision <= 18; precision < 10 will produce a warning

This PR implements the `precision < 10 will produce a warning` part.

@rdblue @liancheng would mind taking a look at this when you have time? It's a fairly small
addition; cheers.

Author: Liwei Lin <proflin.me@gmail.com>
Author: proflin <proflin.me@gmail.com>

Closes #316 from lw-lin/P-484-2 and squashes the following commits:

207e509 [Liwei Lin] Address comments
b227484 [proflin] PARQUET-484: Warn when Decimal is stored as INT64 while could be stored
as INT32


Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/aa249a5a
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/aa249a5a
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/aa249a5a

Branch: refs/heads/parquet-1.8.x
Commit: aa249a5ab9039d7fa3a2e3e1dfa1b0934844cd1e
Parents: de17dea
Author: Liwei Lin <proflin.me@gmail.com>
Authored: Tue Apr 19 09:17:01 2016 -0700
Committer: Ryan Blue <blue@apache.org>
Committed: Mon Jan 9 16:54:53 2017 -0800

----------------------------------------------------------------------
 .../src/main/java/org/apache/parquet/schema/Types.java      | 9 +++++++++
 1 file changed, 9 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/aa249a5a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
index 6d61b50..d3cc97b 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
@@ -25,6 +25,8 @@ import java.util.List;
 import org.apache.parquet.Preconditions;
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
 import org.apache.parquet.schema.Type.ID;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class provides fluent builders that produce Parquet schema Types.
@@ -305,8 +307,11 @@ public class Types {
   public abstract static class
       BasePrimitiveBuilder<P, THIS extends BasePrimitiveBuilder<P, THIS>>
       extends Builder<THIS, P> {
+    private static final Logger LOGGER = LoggerFactory.getLogger(BasePrimitiveBuilder.class);
     private static final long MAX_PRECISION_INT32 = maxPrecision(4);
     private static final long MAX_PRECISION_INT64 = maxPrecision(8);
+    private static final String LOGICAL_TYPES_DOC_URL =
+        "https://github.com/apache/parquet-format/blob/master/LogicalTypes.md";
     private final PrimitiveTypeName primitiveType;
     private int length = NOT_SET;
     private int precision = NOT_SET;
@@ -406,6 +411,10 @@ public class Types {
                   meta.getPrecision() <= MAX_PRECISION_INT64,
                   "INT64 cannot store " + meta.getPrecision() + " digits " +
                   "(max " + MAX_PRECISION_INT64 + ")");
+              if (meta.getPrecision() <= MAX_PRECISION_INT32) {
+                LOGGER.warn("Decimal with {} digits is stored in an INT64, but fits in an
INT32. See {}.",
+                            precision, LOGICAL_TYPES_DOC_URL);
+              }
             } else if (primitiveType == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
               Preconditions.checkState(
                   meta.getPrecision() <= maxPrecision(length),


Mime
View raw message