carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [17/51] [abbrv] carbondata git commit: [CARBONDATA-1470] csv data should not show in error log when data column length is greater than 100000 characters
Date Mon, 18 Sep 2017 10:01:36 GMT
[CARBONDATA-1470] csv data should not show in error log when data column length is greater
than 100000 characters

Added method to handle TextParsingException and stop the leak of the sensitive data in logs

This closes #1349


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/33ecca9b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/33ecca9b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/33ecca9b

Branch: refs/heads/branch-1.2
Commit: 33ecca9b7c256285b2f10a080b1c6bae14fef5a3
Parents: 1551a7c
Author: kapilreja <kapilreja@gmail.com>
Authored: Tue Sep 12 06:32:36 2017 +0000
Committer: Ravindra Pesala <ravi.pesala@gmail.com>
Committed: Tue Sep 12 15:34:31 2017 +0530

----------------------------------------------------------------------
 .../processing/csvload/CSVRecordReaderIterator.java |  8 ++++++++
 .../processing/util/CarbonDataProcessorUtil.java    | 16 ++++++++++++++++
 2 files changed, 24 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/33ecca9b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java
b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java
index 10a036a..efe75ef 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java
@@ -19,8 +19,12 @@ package org.apache.carbondata.processing.csvload;
 
 import java.io.IOException;
 
+
 import org.apache.carbondata.common.CarbonIterator;
 import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
+
+import com.univocity.parsers.common.TextParsingException;
 
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.mapreduce.InputSplit;
@@ -62,6 +66,10 @@ public class CSVRecordReaderIterator extends CarbonIterator<Object []>
{
       }
       return true;
     } catch (Exception e) {
+      if (e instanceof TextParsingException) {
+        throw new CarbonDataLoadingException(
+            CarbonDataProcessorUtil.trimErrorMessage(e.getMessage()));
+      }
       throw new CarbonDataLoadingException(e);
     }
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/33ecca9b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 45461e3..e91cf44 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -610,4 +610,20 @@ public final class CarbonDataProcessorUtil {
     }
     return outArr;
   }
+
+  /**
+   * This method returns String if exception is TextParsingException
+   *
+   * @param input
+   * @return
+   */
+  public static String trimErrorMessage(String input) {
+    String errorMessage = input;
+    if (input != null) {
+      if (input.split("Hint").length > 0) {
+        errorMessage = input.split("Hint")[0];
+      }
+    }
+    return errorMessage;
+  }
 }
\ No newline at end of file


Mime
View raw message