Repository: parquet-mr
Updated Branches:
refs/heads/master dc08bb8ea -> 82b8ecc32
PARQUET-484: Warn when Decimal is stored as INT64 while could be stored as INT32
Below is documented in [LogicalTypes.md](https://github.com/Parquet/parquet-format/blob/master/LogicalTypes.md#decimal):
> int32: for 1 <= precision <= 9
> int64: for 1 <= precision <= 18; precision < 10 will produce a warning
This PR implements the `precision < 10 will produce a warning` part.
@rdblue @liancheng would mind taking a look at this when you have time? It's a fairly small
addition; cheers.
Author: Liwei Lin <proflin.me@gmail.com>
Author: proflin <proflin.me@gmail.com>
Closes #316 from lw-lin/P-484-2 and squashes the following commits:
207e509 [Liwei Lin] Address comments
b227484 [proflin] PARQUET-484: Warn when Decimal is stored as INT64 while could be stored
as INT32
Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/82b8ecc3
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/82b8ecc3
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/82b8ecc3
Branch: refs/heads/master
Commit: 82b8ecc3275d7c3578a6531ac3f1da3ffada9dcc
Parents: dc08bb8
Author: Liwei Lin <proflin.me@gmail.com>
Authored: Tue Apr 19 09:17:01 2016 -0700
Committer: Ryan Blue <blue@apache.org>
Committed: Tue Apr 19 09:17:01 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/parquet/schema/Types.java | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/82b8ecc3/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
index 6d61b50..d3cc97b 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
@@ -25,6 +25,8 @@ import java.util.List;
import org.apache.parquet.Preconditions;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Type.ID;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class provides fluent builders that produce Parquet schema Types.
@@ -305,8 +307,11 @@ public class Types {
public abstract static class
BasePrimitiveBuilder<P, THIS extends BasePrimitiveBuilder<P, THIS>>
extends Builder<THIS, P> {
+ private static final Logger LOGGER = LoggerFactory.getLogger(BasePrimitiveBuilder.class);
private static final long MAX_PRECISION_INT32 = maxPrecision(4);
private static final long MAX_PRECISION_INT64 = maxPrecision(8);
+ private static final String LOGICAL_TYPES_DOC_URL =
+ "https://github.com/apache/parquet-format/blob/master/LogicalTypes.md";
private final PrimitiveTypeName primitiveType;
private int length = NOT_SET;
private int precision = NOT_SET;
@@ -406,6 +411,10 @@ public class Types {
meta.getPrecision() <= MAX_PRECISION_INT64,
"INT64 cannot store " + meta.getPrecision() + " digits " +
"(max " + MAX_PRECISION_INT64 + ")");
+ if (meta.getPrecision() <= MAX_PRECISION_INT32) {
+ LOGGER.warn("Decimal with {} digits is stored in an INT64, but fits in an
INT32. See {}.",
+ precision, LOGICAL_TYPES_DOC_URL);
+ }
} else if (primitiveType == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
Preconditions.checkState(
meta.getPrecision() <= maxPrecision(length),
|