carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [10/42] carbondata git commit: [CARBONDATA-989] decompress error while load 'gz' and 'bz2' data into table
Date Thu, 15 Jun 2017 11:50:15 GMT
[CARBONDATA-989] decompress error while load 'gz' and 'bz2' data into table


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/59d55454
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/59d55454
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/59d55454

Branch: refs/heads/branch-1.1
Commit: 59d55454c2bd606bb59c6ea9997191fa38480916
Parents: 9e913e0
Author: ranmx <ranmx@fosun.com>
Authored: Thu Apr 27 16:17:33 2017 +0800
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Thu Jun 15 12:45:27 2017 +0530

----------------------------------------------------------------------
 .../core/datastore/impl/FileFactory.java        |  12 +-
 .../core/datastore/CompressdFileTest.java       | 120 +++++++++++++++++++
 2 files changed, 129 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/59d55454/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
index c32b956..de78a3f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
 import org.apache.hadoop.io.compress.GzipCodec;
 
 public final class FileFactory {
@@ -131,11 +133,15 @@ public final class FileFactory {
         } else {
           stream = fs.open(pt, bufferSize);
         }
+        String codecName = null;
         if (gzip) {
-          GzipCodec codec = new GzipCodec();
-          stream = codec.createInputStream(stream);
+          codecName = GzipCodec.class.getName();
         } else if (bzip2) {
-          BZip2Codec codec = new BZip2Codec();
+          codecName = BZip2Codec.class.getName();
+        }
+        if (null != codecName) {
+          CompressionCodecFactory ccf = new CompressionCodecFactory(configuration);
+          CompressionCodec codec = ccf.getCodecByClassName(codecName);
           stream = codec.createInputStream(stream);
         }
         break;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/59d55454/core/src/test/java/org/apache/carbondata/core/datastore/CompressdFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/CompressdFileTest.java
b/core/src/test/java/org/apache/carbondata/core/datastore/CompressdFileTest.java
new file mode 100644
index 0000000..14aec44
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/CompressdFileTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory.*;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+
+import java.io.DataInputStream;
+import java.io.FileOutputStream;
+import java.io.File;
+import java.io.Writer;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import static junit.framework.TestCase.assertTrue;
+
+
+public class CompressdFileTest
+{
+  @BeforeClass public static void setUp() throws Exception  {
+    String path = "../core/src/test/resources/testFile";
+    String content = "hello world";
+
+    makeGzipFile(path, content);
+    makeBzip2File(path, content);
+
+  }
+
+  private static void makeGzipFile (String path, String content) throws Exception {
+    path = path + ".gz";
+    FileOutputStream output = new FileOutputStream(path);
+    try {
+      Writer writer = new OutputStreamWriter(new GZIPOutputStream(output),
+          "UTF-8");
+      try {
+        writer.write(content);
+      } finally {
+        writer.close();
+      }
+    } finally {
+      output.close();
+  }
+}
+
+  private static void makeBzip2File (String path, String content) throws Exception {
+    path = path + ".bz2";
+    FileOutputStream output = new FileOutputStream(path);
+    try {
+      Writer writer = new OutputStreamWriter(new BZip2CompressorOutputStream(output),
+          "UTF-8");
+      try {
+        writer.write(content);
+      } finally {
+        writer.close();
+      }
+    } finally {
+      output.close();
+    }
+  }
+
+  @Test public void testReadGzFile() throws Exception {
+    assertTrue(readCompressed("../core/src/test/resources/testFile.gz").equals("hello world"));
+  }
+
+  @Test public void testReadBzip2File() throws Exception {
+    assertTrue(readCompressed("../core/src/test/resources/testFile.bz2").equals("hello world"));
+  }
+
+  private static String readCompressed(String path) throws Exception {
+      DataInputStream fileReader = null;
+      BufferedReader bufferedReader = null;
+      String readLine = null;
+
+      try {
+        fileReader =
+            FileFactory.getDataInputStream(path, FileType.HDFS);
+        bufferedReader = new BufferedReader(new InputStreamReader(fileReader,
+            Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
+        readLine = bufferedReader.readLine();
+      } finally {
+        if (null != fileReader) {
+          fileReader.close();
+        }
+
+        if (null != bufferedReader) {
+          bufferedReader.close();
+        }
+      }
+      return readLine;
+  }
+
+  @AfterClass public static void testCleanUp() {
+    new File("../core/src/test/resources/testFile.gz").deleteOnExit();
+    new File("../core/src/test/resources/testFile.bz2").deleteOnExit();
+  }
+}


Mime
View raw message