carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [1/2] carbondata git commit: [CARBONDATA-1049] Avoid logging raw data into driver and executor log.// added isDebuggEnabled check
Date Fri, 09 Jun 2017 13:30:02 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 657902d7d -> f53ab4b2b


[CARBONDATA-1049] Avoid logging raw data into driver and executor log.// added isDebuggEnabled
check

[CARBONDATA-1049] Avoid logging raw data into driver and executor log.


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4a79a86a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4a79a86a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4a79a86a

Branch: refs/heads/master
Commit: 4a79a86ab06a3b053ac01094a2525a534a639d3a
Parents: 657902d
Author: mohammadshahidkhan <mohdshahidkhan1987@gmail.com>
Authored: Fri May 12 12:36:42 2017 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Fri Jun 9 18:59:19 2017 +0530

----------------------------------------------------------------------
 .../carbondata/core/cache/CarbonLRUCache.java   | 10 +++++---
 .../DateDirectDictionaryGenerator.java          | 13 ++++++----
 .../TimeStampDirectDictionaryGenerator.java     | 15 +++++++-----
 .../core/mutate/DeleteDeltaBlockDetails.java    |  4 +++-
 .../core/mutate/SegmentUpdateDetails.java       |  6 +++--
 .../data/BlockletDeleteDeltaCacheLoader.java    |  8 +++++--
 .../scan/filter/FilterExpressionProcessor.java  | 19 ++++++++-------
 .../carbondata/core/scan/filter/FilterUtil.java |  4 +++-
 .../carbondata/core/util/DataTypeUtil.java      |  8 +++----
 .../badrecordloger/BadRecordLoggerTest.scala    |  2 +-
 .../newflow/converter/BadRecordLogHolder.java   | 21 ++++++++++++++++
 .../impl/DictionaryFieldConverterImpl.java      | 12 ++++++----
 .../DirectDictionaryFieldConverterImpl.java     | 18 +++++++-------
 .../impl/MeasureFieldConverterImpl.java         | 25 +++++++++++++-------
 .../impl/NonDictionaryFieldConverterImpl.java   | 12 ++++++----
 .../converter/impl/RowConverterImpl.java        |  8 +++++--
 .../unsafe/merger/UnsafeIntermediateMerger.java | 10 +++++---
 .../sortandgroupby/sortdata/SortDataRows.java   |  8 +++++--
 .../sortdata/SortIntermediateFileMerger.java    |  4 +++-
 .../store/writer/AbstractFactDataWriter.java    |  6 +++--
 .../util/CarbonDataProcessorUtil.java           | 12 ++++++++++
 21 files changed, 156 insertions(+), 69 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java b/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java
index 151ec58..23dd814 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java
@@ -171,8 +171,10 @@ public final class CarbonLRUCache {
    * @param cacheInfo
    */
   public boolean put(String columnIdentifier, Cacheable cacheInfo, long requiredSize) {
-    LOGGER.debug("Required size for entry " + columnIdentifier + " :: " + requiredSize
-        + " Current cache size :: " + currentSize);
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug("Required size for entry " + columnIdentifier + " :: " + requiredSize
+          + " Current cache size :: " + currentSize);
+    }
     boolean columnKeyAddedSuccessfully = false;
     if (isLRUCacheSizeConfigured()) {
       synchronized (lruCacheMap) {
@@ -206,7 +208,9 @@ public final class CarbonLRUCache {
     if (null == lruCacheMap.get(columnIdentifier)) {
       lruCacheMap.put(columnIdentifier, cacheInfo);
     }
-    LOGGER.debug("Added entry to InMemory lru cache :: " + columnIdentifier);
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug("Added entry to InMemory lru cache :: " + columnIdentifier);
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index a21fd85..9fc2c25 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -109,9 +109,10 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
       }
       dateToStr = simpleDateFormat.parse(memberStr);
     } catch (ParseException e) {
-      LOGGER.debug(
-          "Cannot convert " + memberStr + " to Time/Long type value. Value considered as
null." + e
-              .getMessage());
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug("Cannot convert value to Time/Long type value. Value considered as null."
+ e
+            .getMessage());
+      }
       dateToStr = null;
     }
     //adding +2 to reserve the first cuttOffDiff value for null or empty date
@@ -140,8 +141,10 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
     try {
       timeValue = Long.valueOf(memberStr) / 1000;
     } catch (NumberFormatException e) {
-      LOGGER.debug("Cannot convert " + memberStr + " Long type value. Value considered as
null." + e
-          .getMessage());
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug(
+            "Cannot convert value to Long type value. Value considered as null." + e.getMessage());
+      }
     }
     if (timeValue == -1) {
       return 1;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index 7d3d6c8..5c7ba07 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -163,9 +163,10 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
       }
       dateToStr = simpleDateFormat.parse(memberStr);
     } catch (ParseException e) {
-      LOGGER.debug(
-          "Cannot convert " + memberStr + " to Time/Long type value. Value considered as
null." + e
-              .getMessage());
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug("Cannot convert value to Time/Long type value. Value considered as null."
+ e
+            .getMessage());
+      }
       dateToStr = null;
     }
     //adding +2 to reserve the first cuttOffDiff value for null or empty date
@@ -200,9 +201,11 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
     try {
       timeValue = Long.valueOf(memberStr) / 1000;
     } catch (NumberFormatException e) {
-      LOGGER.debug(
-          "Cannot convert " + memberStr + " Long type value. Value considered as null." +
e
-              .getMessage());
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug(
+            "Cannot convert " + memberStr + " Long type value. Value considered as null."
+ e
+                .getMessage());
+      }
     }
     if (timeValue == -1) {
       return 1;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
index c4e9ea2..b1293e1 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
@@ -86,7 +86,9 @@ public class DeleteDeltaBlockDetails implements Serializable {
       blocklet.addDeletedRow(CarbonUpdateUtil.getIntegerValue(offset));
       return addBlockletDetails(blocklet);
     } catch (Exception e) {
-      LOGGER.debug(e.getMessage());
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug(e.getMessage());
+      }
       throw e;
     }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/mutate/SegmentUpdateDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/SegmentUpdateDetails.java
b/core/src/main/java/org/apache/carbondata/core/mutate/SegmentUpdateDetails.java
index 6f7eef4..6f99b3a 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/SegmentUpdateDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/SegmentUpdateDetails.java
@@ -158,8 +158,10 @@ public class SegmentUpdateDetails implements Serializable {
     try {
       longValue = Long.parseLong(timtstamp);
     } catch (NumberFormatException nfe) {
-      String errorMsg = "Invalid timestamp : " + timtstamp;
-      LOGGER.debug(errorMsg);
+      if (LOGGER.isDebugEnabled()) {
+        String errorMsg = "Invalid timestamp : " + timtstamp;
+        LOGGER.debug(errorMsg);
+      }
     }
     return longValue;
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
b/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
index 6665c5b..93607cc 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
@@ -57,7 +57,9 @@ public class BlockletDeleteDeltaCacheLoader implements DeleteDeltaCacheLoaderInt
         deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData,
             segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, null));
       } catch (Exception e) {
-        LOGGER.debug("Unable to retrieve delete delta files");
+        if (LOGGER.isDebugEnabled()) {
+          LOGGER.debug("Unable to retrieve delete delta files");
+        }
       }
     } else {
       deleteDeltaDataCache = blockletNode.getDeleteDeltaDataCache();
@@ -71,7 +73,9 @@ public class BlockletDeleteDeltaCacheLoader implements DeleteDeltaCacheLoaderInt
           deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData,
               segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, cacheTimeStamp));
         } catch (Exception e) {
-          LOGGER.debug("Unable to retrieve delete delta files");
+          if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug("Unable to retrieve delete delta files");
+          }
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 5e69f3a..ce31283 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -107,8 +107,10 @@ public class FilterExpressionProcessor implements FilterProcessor {
     List<DataRefNode> listOfDataBlocksToScan = new ArrayList<DataRefNode>();
     // getting the start and end index key based on filter for hitting the
     // selected block reference nodes based on filter resolver tree.
-    LOGGER.debug("preparing the start and end key for finding"
-        + "start and end block as per filter resolver");
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug("preparing the start and end key for finding"
+          + "start and end block as per filter resolver");
+    }
     List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
     FilterUtil.traverseResolverTreeAndGetStartAndEndKey(tableSegment.getSegmentProperties(),
         filterResolver, listOfStartEndKeys);
@@ -127,12 +129,13 @@ public class FilterExpressionProcessor implements FilterProcessor {
         return listOfDataBlocksToScan;
       }
     }
-
-    LOGGER.debug(
-        "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey
-            .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey.getNoDictionaryKeys()
-            + "Dictionary End Key: " + searchEndKey.getDictionaryKeys() + "No Dictionary
End Key "
-            + searchEndKey.getNoDictionaryKeys());
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug(
+          "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey
+              .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey
+              .getNoDictionaryKeys() + "Dictionary End Key: " + searchEndKey.getDictionaryKeys()
+              + "No Dictionary End Key " + searchEndKey.getNoDictionaryKeys());
+    }
     long startTimeInMillis = System.currentTimeMillis();
     DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
         tableSegment.getSegmentProperties().getEachDimColumnValueSize(),

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 5434a5f..b06789e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -503,7 +503,9 @@ public final class FilterUtil {
             }
           }
         } catch (FilterIllegalMemberException e) {
-          LOGGER.debug(e.getMessage());
+          if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(e.getMessage());
+          }
         }
       }
       return getFilterValues(columnExpression, evaluateResultListFinal, forwardDictionary,

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 596f057..af3ff2f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -287,7 +287,7 @@ public final class DataTypeUtil {
             Date dateToStr = dateformatter.get().parse(data);
             return dateToStr.getTime() * 1000;
           } catch (ParseException e) {
-            LOGGER.error("Cannot convert" + data + " to Time/Long type value" + e.getMessage());
+            LOGGER.error("Cannot convert value to Time/Long type value" + e.getMessage());
             return null;
           }
 
@@ -299,7 +299,7 @@ public final class DataTypeUtil {
             Date dateToStr = timeStampformatter.get().parse(data);
             return dateToStr.getTime() * 1000;
           } catch (ParseException e) {
-            LOGGER.error("Cannot convert" + data + " to Time/Long type value" + e.getMessage());
+            LOGGER.error("Cannot convert value to Time/Long type value" + e.getMessage());
             return null;
           }
         case DECIMAL:
@@ -436,7 +436,7 @@ public final class DataTypeUtil {
             Date dateToStr = dateformatter.get().parse(data5);
             return dateToStr.getTime() * 1000;
           } catch (ParseException e) {
-            LOGGER.error("Cannot convert" + data5 + " to Time/Long type value" + e.getMessage());
+            LOGGER.error("Cannot convert value to Time/Long type value" + e.getMessage());
             return null;
           }
 
@@ -449,7 +449,7 @@ public final class DataTypeUtil {
             Date dateToStr = timeStampformatter.get().parse(data6);
             return dateToStr.getTime() * 1000;
           } catch (ParseException e) {
-            LOGGER.error("Cannot convert" + data6 + " to Time/Long type value" + e.getMessage());
+            LOGGER.error("Cannot convert value to Time/Long type value" + e.getMessage());
             return null;
           }
         case DECIMAL:

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala
index 5716978..1c624bf 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala
@@ -193,7 +193,7 @@ class BadRecordLoggerTest extends QueryTest with BeforeAndAfterAll {
   test("select count(*) from insufficientColumn") {
     checkAnswer(
       sql("select count(*) from insufficientColumn"),
-      Seq(Row(1)
+      Seq(Row(3)
       )
     )
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/BadRecordLogHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/BadRecordLogHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/BadRecordLogHolder.java
index 42a6fec..f7ce620 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/BadRecordLogHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/BadRecordLogHolder.java
@@ -16,11 +16,19 @@
  */
 package org.apache.carbondata.processing.newflow.converter;
 
+import java.util.HashMap;
+import java.util.Map;
+
 /**
  * It is holder for reason of bad records.
  */
 public class BadRecordLogHolder {
 
+  /**
+   * this map will hold the bad record unified message for columns
+   */
+  private Map<String, String> columnMessageMap = new HashMap<>();
+
   private String reason;
 
   private boolean badRecordAdded;
@@ -51,4 +59,17 @@ public class BadRecordLogHolder {
   public void setLogged(boolean logged) {
     isLogged = logged;
   }
+
+  public Map<String, String> getColumnMessageMap() {
+    return columnMessageMap;
+  }
+
+  /**
+   * this method will clear the map entries
+   */
+  public void finish() {
+    if (null != columnMessageMap) {
+      columnMessageMap.clear();
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
index 23e9c8c..81b44f3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
@@ -40,6 +40,7 @@ import org.apache.carbondata.processing.newflow.dictionary.DictionaryServerClien
 import org.apache.carbondata.processing.newflow.dictionary.PreCreatedDictionary;
 import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
 import org.apache.carbondata.processing.newflow.row.CarbonRow;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
 
 public class DictionaryFieldConverterImpl extends AbstractDictionaryFieldConverterImpl {
 
@@ -106,11 +107,12 @@ public class DictionaryFieldConverterImpl extends AbstractDictionaryFieldConvert
       }
       if (null == parsedValue) {
         if ((dimensionValue.length() > 0) || (dimensionValue.length() == 0 &&
isEmptyBadRecord)) {
-          String dataType = carbonDimension.getDataType().getName();
-          logHolder.setReason(
-              "The value " + " \"" + dimensionValue + "\"" + " with column name " + carbonDimension
-                  .getColName() + " and column data type " + dataType + " is not a valid
"
-                  + dataType + " type.");
+          String message = logHolder.getColumnMessageMap().get(carbonDimension.getColName());
+          if (null == message) {
+            message = CarbonDataProcessorUtil
+                .prepareFailureReason(carbonDimension.getColName(), carbonDimension.getDataType());
+            logHolder.getColumnMessageMap().put(carbonDimension.getColName(), message);
+          } logHolder.setReason(message);
         }
         row.update(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY, index);
       } else {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java
index 05ffe8b..39e721b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java
@@ -25,6 +25,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
 import org.apache.carbondata.processing.newflow.DataField;
 import org.apache.carbondata.processing.newflow.converter.BadRecordLogHolder;
 import org.apache.carbondata.processing.newflow.row.CarbonRow;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
 
 public class DirectDictionaryFieldConverterImpl extends AbstractDictionaryFieldConverterImpl
{
 
@@ -58,10 +59,8 @@ public class DirectDictionaryFieldConverterImpl extends AbstractDictionaryFieldC
   public void convert(CarbonRow row, BadRecordLogHolder logHolder) {
     String value = row.getString(index);
     if (value == null) {
-      logHolder.setReason(
-          "The value " + " \"" + row.getString(index) + "\"" + " with column name " + column
-              .getColName() + " and column data type " + column.getDataType() + " is not
a valid "
-              + column.getDataType() + " type.");
+      logHolder.setReason(CarbonDataProcessorUtil
+          .prepareFailureReason(column.getColName(), column.getDataType()));
       row.update(1, index);
     } else if (value.equals(nullFormat)) {
       row.update(1, index);
@@ -69,10 +68,13 @@ public class DirectDictionaryFieldConverterImpl extends AbstractDictionaryFieldC
       int key = directDictionaryGenerator.generateDirectSurrogateKey(value);
       if (key == 1) {
         if ((value.length() > 0) || (value.length() == 0 && isEmptyBadRecord))
{
-          logHolder.setReason(
-              "The value " + " \"" + row.getString(index) + "\"" + " with column name " +
column
-                  .getColName() + " and column data type " + column.getDataType()
-                  + " is not a valid " + column.getDataType() + " type.");
+          String message = logHolder.getColumnMessageMap().get(column.getColName());
+          if (null == message) {
+            message = CarbonDataProcessorUtil
+                .prepareFailureReason(column.getColName(), column.getDataType());
+            logHolder.getColumnMessageMap().put(column.getColName(), message);
+          }
+          logHolder.setReason(message);
         }
       }
       row.update(key, index);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java
index bb5f983..01ea437 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java
@@ -27,6 +27,7 @@ import org.apache.carbondata.processing.newflow.converter.BadRecordLogHolder;
 import org.apache.carbondata.processing.newflow.converter.FieldConverter;
 import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
 import org.apache.carbondata.processing.newflow.row.CarbonRow;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
 
 /**
  * Converter for measure
@@ -62,15 +63,22 @@ public class MeasureFieldConverterImpl implements FieldConverter {
     Object output;
     boolean isNull = CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(value);
     if (value == null || isNull) {
-      logHolder.setReason(
-          "The value " + " \"" + value + "\"" + " with column name " + measure.getColName()
-              + " and column data type " + dataType + " is not a valid " + dataType + " type.");
+      String message = logHolder.getColumnMessageMap().get(measure.getColName());
+      if (null == message) {
+        value = CarbonDataProcessorUtil
+            .prepareFailureReason(measure.getColName(), measure.getDataType());
+        logHolder.getColumnMessageMap().put(measure.getColName(), message);
+      }
       row.update(null, index);
     } else if (value.length() == 0) {
       if (isEmptyBadRecord) {
-        logHolder.setReason(
-            "The value " + " \"" + value + "\"" + " with column name " + measure.getColName()
-                + " and column data type " + dataType + " is not a valid " + dataType + "
type.");
+        String message = logHolder.getColumnMessageMap().get(measure.getColName());
+        if (null == message) {
+          message = CarbonDataProcessorUtil
+              .prepareFailureReason(measure.getColName(), measure.getDataType());
+          logHolder.getColumnMessageMap().put(measure.getColName(), message);
+        }
+        logHolder.setReason(message);
       }
       row.update(null, index);
     } else if (value.equals(nullformat)) {
@@ -81,10 +89,9 @@ public class MeasureFieldConverterImpl implements FieldConverter {
         row.update(output, index);
       } catch (NumberFormatException e) {
         LOGGER.warn(
-            "Cant not convert : " + value + " to Numeric type value. Value considered as
null.");
+            "Cant not convert value to Numeric type value. Value considered as null.");
         logHolder.setReason(
-            "The value " + " \"" + value + "\"" + " with column name " + measure.getColName()
-                + " and column data type " + dataType + " is not a valid " + dataType + "
type.");
+            CarbonDataProcessorUtil.prepareFailureReason(measure.getColName(), dataType));
         output = null;
         row.update(output, index);
       }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
index b39ef11..0f88d74 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
@@ -24,6 +24,7 @@ import org.apache.carbondata.processing.newflow.DataField;
 import org.apache.carbondata.processing.newflow.converter.BadRecordLogHolder;
 import org.apache.carbondata.processing.newflow.converter.FieldConverter;
 import org.apache.carbondata.processing.newflow.row.CarbonRow;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
 
 public class NonDictionaryFieldConverterImpl implements FieldConverter {
 
@@ -56,10 +57,13 @@ public class NonDictionaryFieldConverterImpl implements FieldConverter
{
             .getBytesBasedOnDataTypeForNoDictionaryColumn(dimensionValue, dataType), index);
       } catch (Throwable ex) {
         if (dimensionValue.length() > 0 || isEmptyBadRecord) {
-          logHolder.setReason(
-              "The value " + " \"" + dimensionValue + "\"" + " with column name " + column
-                  .getColName() + " and column data type " + dataType + " is not a valid
"
-                  + dataType + " type.");
+          String message = logHolder.getColumnMessageMap().get(column.getColName());
+          if (null == message) {
+            message = CarbonDataProcessorUtil
+                .prepareFailureReason(column.getColName(), column.getDataType());
+            logHolder.getColumnMessageMap().put(column.getColName(), message);
+          }
+          logHolder.setReason(message);
           row.update(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, index);
         } else {
           row.update(new byte[0], index);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
index 4cb7a06..2414d58 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
@@ -156,11 +156,12 @@ public class RowConverterImpl implements RowConverter {
     for (int i = 0; i < fieldConverters.length; i++) {
       fieldConverters[i].convert(row, logHolder);
       if (!logHolder.isLogged() && logHolder.isBadRecordNotAdded()) {
+        badRecordLogger.addBadRecordsToBuilder(copy.getData(), logHolder.getReason());
         if (badRecordLogger.isDataLoadFail()) {
-          String error = "Data load failed due to bad record: " + logHolder.getReason();
+          String error = "Data load failed due to bad record: " + logHolder.getReason() +
+              "Please enable bad record logger to know the detail reason.";
           throw new BadRecordFoundException(error);
         }
-        badRecordLogger.addBadRecordsToBuilder(copy.getData(), logHolder.getReason());
         logHolder.clear();
         logHolder.setLogged(true);
         if (badRecordLogger.isBadRecordConvertNullDisable()) {
@@ -180,6 +181,9 @@ public class RowConverterImpl implements RowConverter {
           client.shutDown();
         }
       }
+      if (null != logHolder) {
+        logHolder.finish();
+      }
       if (executorService != null) {
         executorService.shutdownNow();
         executorService = null;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateMerger.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateMerger.java
index d7b7b0b..d10c75f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateMerger.java
@@ -93,7 +93,9 @@ public class UnsafeIntermediateMerger {
         fileList = procFiles.toArray(new File[procFiles.size()]);
         this.procFiles = new ArrayList<File>();
       }
-      LOGGER.debug("Sumitting request for intermediate merging no of files: " + fileList.length);
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug("Sumitting request for intermediate merging no of files: " + fileList.length);
+      }
       startIntermediateMerging(fileList);
     }
   }
@@ -124,8 +126,10 @@ public class UnsafeIntermediateMerger {
         localRowPages = rowPages.toArray(new UnsafeCarbonRowPage[rowPages.size()]);
         this.rowPages = new ArrayList<>();
       }
-      LOGGER.debug("Sumitting request for intermediate merging of in-memory pages : "
-          + localRowPages.length);
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug("Sumitting request for intermediate merging of in-memory pages : "
+            + localRowPages.length);
+      }
       startIntermediateMerging(localRowPages, totalRows);
     }
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
index e840984..5df3112 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
@@ -121,7 +121,9 @@ public class SortDataRows {
     int currentSize = entryCount;
 
     if (sortBufferSize == currentSize) {
-      LOGGER.debug("************ Writing to temp file ********** ");
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug("************ Writing to temp file ********** ");
+      }
       intermediateFileMerger.startMergingIfPossible();
       Object[][] recordHolderListLocal = recordHolderList;
       try {
@@ -151,7 +153,9 @@ public class SortDataRows {
     synchronized (addRowsLock) {
       int sizeLeft = 0;
       if (entryCount + size >= sortBufferSize) {
-        LOGGER.debug("************ Writing to temp file ********** ");
+        if (LOGGER.isDebugEnabled()) {
+          LOGGER.debug("************ Writing to temp file ********** ");
+        }
         intermediateFileMerger.startMergingIfPossible();
         Object[][] recordHolderListLocal = recordHolderList;
         sizeLeft = sortBufferSize - entryCount ;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortIntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortIntermediateFileMerger.java
b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortIntermediateFileMerger.java
index 2655a7b..f940375 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortIntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortIntermediateFileMerger.java
@@ -71,7 +71,9 @@ public class SortIntermediateFileMerger {
         fileList = procFiles.toArray(new File[procFiles.size()]);
         this.procFiles = new ArrayList<File>();
       }
-      LOGGER.debug("Sumitting request for intermediate merging no of files: " + fileList.length);
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug("Sumitting request for intermediate merging no of files: " + fileList.length);
+      }
       startIntermediateMerging(fileList);
     }
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 9c7a5ff..0fc1d64 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -538,8 +538,10 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
     DataOutputStream dataOutputStream = null;
     DataInputStream dataInputStream = null;
     try {
-      LOGGER.debug(
-          "HDFS file block size for file: " + carbonStoreFilePath + " is " + blockSize +
" (bytes");
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug("HDFS file block size for file: " + carbonStoreFilePath + " is " + blockSize
+            + " (bytes");
+      }
       dataOutputStream = FileFactory
           .getDataOutputStream(carbonStoreFilePath, FileFactory.getFileType(carbonStoreFilePath),
               bufferSize, blockSize);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 740b5dc..2614b8f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -475,4 +475,16 @@ public final class CarbonDataProcessorUtil {
     return batchSortSizeInMb;
   }
 
+  /**
+   * the method prepares and return the message mentioning the reason of badrecord
+   *
+   * @param columnName
+   * @param dataType
+   * @return
+   */
+  public static String prepareFailureReason(String columnName, DataType dataType) {
+    return "The value with column name " + columnName + " and column data type " + dataType
+        .getName() + " is not a valid " + dataType + " type.";
+  }
+
 }
\ No newline at end of file


Mime
View raw message