parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jul...@apache.org
Subject parquet-mr git commit: PARQUET-969: Update parquet-tools to convert Decimal datatype to BigD…
Date Fri, 12 May 2017 21:40:34 GMT
Repository: parquet-mr
Updated Branches:
  refs/heads/master 70f28810a -> a703ee75c


PARQUET-969: Update parquet-tools to convert Decimal datatype to BigD…

Update parquet-tools so that decimal datatypes in parquet files are converted to their actual
number representation when cat'ing to stdout. Currently they are output in binary format.

Author: dsfcode <fowler.dn@gmail.com>

Closes #412 from dsfcode/master and squashes the following commits:

7f05509 [dsfcode] PARQUET-969: Update parquet-tools to convert Decimal datatype to BigDecimal


Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/a703ee75
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/a703ee75
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/a703ee75

Branch: refs/heads/master
Commit: a703ee75c40e0207f6831c4d48e1c7e62f160305
Parents: 70f2881
Author: dsfcode <fowler.dn@gmail.com>
Authored: Fri May 12 14:40:29 2017 -0700
Committer: Julien Le Dem <julien@apache.org>
Committed: Fri May 12 14:40:29 2017 -0700

----------------------------------------------------------------------
 .../tools/read/SimpleRecordConverter.java       | 20 +++++++
 .../tools/read/TestSimplePrimitiveRecord.java   | 58 ++++++++++++++++++++
 2 files changed, 78 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/a703ee75/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java
----------------------------------------------------------------------
diff --git a/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java
b/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java
index 5f78444..29bb44f 100644
--- a/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java
+++ b/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java
@@ -22,6 +22,9 @@ import java.nio.CharBuffer;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
 
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
 import org.apache.parquet.io.api.Binary;
 import org.apache.parquet.io.api.Converter;
 import org.apache.parquet.io.api.GroupConverter;
@@ -70,6 +73,9 @@ public class SimpleRecordConverter extends GroupConverter {
           case UTF8: return new StringConverter(field.getName());
           case MAP_KEY_VALUE: break;
           case ENUM: break;
+          case DECIMAL:
+            int scale = field.asPrimitiveType().getDecimalMetadata().getScale();
+            return new DecimalConverter(field.getName(), scale);
         }
       }
 
@@ -155,5 +161,19 @@ public class SimpleRecordConverter extends GroupConverter {
       record.add(name, value.toStringUsingUTF8());
     }
   }
+
+  private class DecimalConverter extends SimplePrimitiveConverter {
+    private final int scale;
+
+    public DecimalConverter(String name, int scale) {
+      super(name);
+      this.scale = scale;
+    }
+
+    @Override
+    public void addBinary(Binary value) {
+      record.add(name, new BigDecimal(new BigInteger(value.getBytes()), scale));
+    }
+  }
 }
 

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/a703ee75/parquet-tools/src/test/java/org/apache/parquet/tools/read/TestSimplePrimitiveRecord.java
----------------------------------------------------------------------
diff --git a/parquet-tools/src/test/java/org/apache/parquet/tools/read/TestSimplePrimitiveRecord.java
b/parquet-tools/src/test/java/org/apache/parquet/tools/read/TestSimplePrimitiveRecord.java
new file mode 100644
index 0000000..de27fcb
--- /dev/null
+++ b/parquet-tools/src/test/java/org/apache/parquet/tools/read/TestSimplePrimitiveRecord.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.tools.read;
+
+import org.junit.Assert;
+import org.junit.Test;
+import java.math.BigDecimal;
+
+public class TestSimplePrimitiveRecord {
+
+  class TestRecord {
+    private int x;
+    private int y;
+
+    public TestRecord(int x, int y) {
+      this.x = x;
+      this.y = y;
+    }
+
+    @Override
+    public String toString() {
+      return "TestRecord {" + x + "," + y + "}";
+    }
+  }
+
+  @Test
+  public void testBinary() {
+    SimpleMapRecord r = new SimpleMapRecord();
+    Assert.assertEquals("null", r.keyToString(null));
+    Assert.assertEquals("true", r.keyToString(true));
+    Assert.assertEquals("a", r.keyToString('a'));
+    Assert.assertEquals("3.0", r.keyToString(3.0));
+    Assert.assertEquals("4.0", r.keyToString(4.0f));
+    Assert.assertEquals("100", r.keyToString(100));
+    Assert.assertEquals("37", r.keyToString(37l));
+    Assert.assertEquals("-1", r.keyToString((short) -1));
+    Assert.assertEquals("test", r.keyToString("test"));
+    Assert.assertEquals("123.123", r.keyToString(new BigDecimal("123.123")));
+  }
+}
+


Mime
View raw message