parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nkol...@apache.org
Subject [parquet-mr] branch master updated: PARQUET-1530: Remove Dependency on commons-codec (#618)
Date Thu, 05 Sep 2019 06:56:37 GMT
This is an automated email from the ASF dual-hosted git repository.

nkollar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git


The following commit(s) were added to refs/heads/master by this push:
     new 340d157  PARQUET-1530: Remove Dependency on commons-codec (#618)
340d157 is described below

commit 340d157bda4c33b7d126cad82e222a2cfb1b2953
Author: belugabehr <12578579+belugabehr@users.noreply.github.com>
AuthorDate: Thu Sep 5 01:56:31 2019 -0500

    PARQUET-1530: Remove Dependency on commons-codec (#618)
---
 parquet-cli/pom.xml                                     |  6 ------
 .../src/main/java/org/apache/parquet/cli/Util.java      | 17 ++++++-----------
 parquet-cli/src/main/resources/META-INF/LICENSE         |  5 ++---
 parquet-column/pom.xml                                  |  6 ------
 parquet-encoding/pom.xml                                |  6 ------
 .../apache/parquet/hadoop/util/SerializationUtil.java   | 15 +++++++--------
 parquet-tools/src/main/resources/META-INF/LICENSE       |  9 ---------
 pom.xml                                                 |  1 -
 8 files changed, 15 insertions(+), 50 deletions(-)

diff --git a/parquet-cli/pom.xml b/parquet-cli/pom.xml
index 5988cab..84bf966 100644
--- a/parquet-cli/pom.xml
+++ b/parquet-cli/pom.xml
@@ -81,12 +81,6 @@
       <scope>provided</scope>
     </dependency>
     <dependency>
-      <groupId>commons-codec</groupId>
-      <artifactId>commons-codec</artifactId>
-      <version>${commons-codec.version}</version>
-      <scope>provided</scope>
-    </dependency>
-    <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client</artifactId>
       <version>${hadoop.version}</version>
diff --git a/parquet-cli/src/main/java/org/apache/parquet/cli/Util.java b/parquet-cli/src/main/java/org/apache/parquet/cli/Util.java
index 961c7f0..cf4b97e 100644
--- a/parquet-cli/src/main/java/org/apache/parquet/cli/Util.java
+++ b/parquet-cli/src/main/java/org/apache/parquet/cli/Util.java
@@ -19,11 +19,13 @@
 
 package org.apache.parquet.cli;
 
+import com.google.common.base.Ascii;
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Splitter;
 import com.google.common.collect.Iterables;
-import org.apache.commons.codec.binary.Hex;
+import com.google.common.hash.HashCode;
+
 import org.apache.parquet.column.ColumnDescriptor;
 import org.apache.parquet.column.Encoding;
 import org.apache.parquet.column.EncodingStats;
@@ -126,20 +128,13 @@ public class Util {
   }
 
   public static String humanReadable(byte[] bytes, int len) {
+    Preconditions.checkArgument(len >= 5, "Display length must be minimum 5");
     if (bytes == null || bytes.length == 0) {
       return "null";
     }
 
-    StringBuilder sb = new StringBuilder();
-    String asString = Hex.encodeHexString(bytes);
-    sb.append("0x");
-    if (asString.length() > len - 2) {
-      sb.append(asString.substring(0, (len - 5) / 2)).append("...");
-    } else {
-      sb.append(asString);
-    }
-
-    return sb.toString();
+    final String asString = HashCode.fromBytes(bytes).toString();
+    return "0x" + Ascii.truncate(asString, len - 2, "...");
   }
 
   public static String shortCodec(CompressionCodecName codec) {
diff --git a/parquet-cli/src/main/resources/META-INF/LICENSE b/parquet-cli/src/main/resources/META-INF/LICENSE
index 2b581f8..3479acc 100644
--- a/parquet-cli/src/main/resources/META-INF/LICENSE
+++ b/parquet-cli/src/main/resources/META-INF/LICENSE
@@ -267,11 +267,10 @@ License: http://www.apache.org/licenses/LICENSE-2.0.txt
 
 --------------------------------------------------------------------------------
 
-This product depends on Apache Commons and includes commons-codec,
-commons-pool, and commons-compress in this binary artifact.
+This product depends on Apache Commons and includes commons-pool, and
+commons-compress in this binary artifact.
 
 Copyright: 2002-2015 The Apache Software Foundation.
-Home page: https://commons.apache.org/proper/commons-codec/
 Home page: https://commons.apache.org/proper/commons-pool/
 License: http://www.apache.org/licenses/LICENSE-2.0
 
diff --git a/parquet-column/pom.xml b/parquet-column/pom.xml
index 8bfba02..b2369b7 100644
--- a/parquet-column/pom.xml
+++ b/parquet-column/pom.xml
@@ -47,12 +47,6 @@
       <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>commons-codec</groupId>
-      <artifactId>commons-codec</artifactId>
-      <version>1.10</version>
-      <scope>compile</scope>
-    </dependency>
-    <dependency>
       <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-encoding</artifactId>
       <version>${project.version}</version>
diff --git a/parquet-encoding/pom.xml b/parquet-encoding/pom.xml
index e3db2ca..6078d7c 100644
--- a/parquet-encoding/pom.xml
+++ b/parquet-encoding/pom.xml
@@ -41,12 +41,6 @@
       <artifactId>parquet-common</artifactId>
       <version>${project.version}</version>
     </dependency>
-    <dependency>
-      <groupId>commons-codec</groupId>
-      <artifactId>commons-codec</artifactId>
-      <version>1.10</version>
-      <scope>compile</scope>
-    </dependency>
 
     <dependency>
       <groupId>org.slf4j</groupId>
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SerializationUtil.java
b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SerializationUtil.java
index 06d5fea..199b774 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SerializationUtil.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SerializationUtil.java
@@ -24,15 +24,12 @@ import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.nio.charset.StandardCharsets;
+import java.util.Base64;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
-import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.conf.Configuration;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 /**
  * Serialization utils copied from:
  * https://github.com/kevinweil/elephant-bird/blob/master/core/src/main/java/com/twitter/elephantbird/util/HadoopUtils.java
@@ -40,7 +37,6 @@ import org.slf4j.LoggerFactory;
  * TODO: Refactor elephant-bird so that we can depend on utils like this without extra baggage.
  */
 public final class SerializationUtil {
-  private static final Logger LOG = LoggerFactory.getLogger(SerializationUtil.class);
 
   private SerializationUtil() { }
 
@@ -58,7 +54,9 @@ public final class SerializationUtil {
             ObjectOutputStream oos = new ObjectOutputStream(gos)) {
         oos.writeObject(obj);
       }
-      conf.set(key, new String(Base64.encodeBase64(baos.toByteArray()), StandardCharsets.UTF_8));
+      conf.set(key,
+          new String(Base64.getMimeEncoder().encode(baos.toByteArray()),
+              StandardCharsets.UTF_8));
     }
   }
 
@@ -79,7 +77,8 @@ public final class SerializationUtil {
       return null;
     }
 
-    byte[] bytes = Base64.decodeBase64(b64.getBytes(StandardCharsets.UTF_8));
+    byte[] bytes =
+        Base64.getMimeDecoder().decode(b64.getBytes(StandardCharsets.UTF_8));
 
     try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
            GZIPInputStream gis = new GZIPInputStream(bais);
@@ -88,7 +87,7 @@ public final class SerializationUtil {
     } catch (ClassNotFoundException e) {
       throw new IOException("Could not read object from config with key " + key, e);
     } catch (ClassCastException e) {
-      throw new IOException("Couldn't cast object read from config with key " + key, e);
+      throw new IOException("Could not cast object read from config with key " + key, e);
     }
   }
 }
diff --git a/parquet-tools/src/main/resources/META-INF/LICENSE b/parquet-tools/src/main/resources/META-INF/LICENSE
index d796f40..126400c 100644
--- a/parquet-tools/src/main/resources/META-INF/LICENSE
+++ b/parquet-tools/src/main/resources/META-INF/LICENSE
@@ -267,15 +267,6 @@ License: http://www.apache.org/licenses/LICENSE-2.0
 
 --------------------------------------------------------------------------------
 
-This product depends on Apache commons-codec and includes it in this binary
-artifact.
-
-Copyright: 2002-2015 The Apache Software Foundation.
-Home page: https://commons.apache.org/proper/commons-codec/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
 This product depends on Google guava and includes it in this binary artifact.
 
 Copyright: 2010-2015 The Guava Authors
diff --git a/pom.xml b/pom.xml
index 1808dba..e1d131e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -105,7 +105,6 @@
     <!-- parquet-cli dependencies -->
     <opencsv.version>2.3</opencsv.version>
     <jcommander.version>1.35</jcommander.version>
-    <commons-codec.version>1.10</commons-codec.version>
   </properties>
 
   <modules>


Mime
View raw message