carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [15/28] carbondata git commit: [CARBONDATA-1741] Remove AKSK in log when saving to S3
Date Sat, 18 Nov 2017 15:29:49 GMT
[CARBONDATA-1741] Remove AKSK in log when saving to S3

This closes #1511


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/75ec79e3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/75ec79e3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/75ec79e3

Branch: refs/heads/fgdatamap
Commit: 75ec79e3f7ddbbfcddb1fbfe1413aae2ee13b6e2
Parents: 52bf7c8
Author: Jacky Li <jacky.likun@qq.com>
Authored: Fri Nov 17 14:57:15 2017 +0800
Committer: QiangCai <qiangcai@qq.com>
Committed: Fri Nov 17 16:22:11 2017 +0800

----------------------------------------------------------------------
 .../apache/carbondata/core/util/CarbonUtil.java  | 19 ++++++++++++++++++-
 .../scala/org/apache/spark/util/FileUtils.scala  |  8 +++++---
 2 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/75ec79e3/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 1e402d0..3c177dc 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -764,6 +764,23 @@ public final class CarbonUtil {
         lowerPath.startsWith(CarbonCommonConstants.S3A_PREFIX);
   }
 
+  public static String removeAKSK(String filePath) {
+    if (null == filePath) {
+      return "";
+    }
+    String lowerPath = filePath.toLowerCase(Locale.getDefault());
+    if (lowerPath.startsWith(CarbonCommonConstants.S3N_PREFIX) ||
+        lowerPath.startsWith(CarbonCommonConstants.S3A_PREFIX) ||
+        lowerPath.startsWith(CarbonCommonConstants.S3_PREFIX)) {
+      int prefixLength = filePath.indexOf(":", 0) + 3;
+      int pathOffset = filePath.indexOf("@");
+      if (pathOffset > 0) {
+        return filePath.substring(0, prefixLength) + filePath.substring(pathOffset + 1);
+      }
+    }
+    return filePath;
+  }
+
   /**
    * This method will check the existence of a file at a given path
    */
@@ -774,7 +791,7 @@ public final class CarbonUtil {
         return true;
       }
     } catch (IOException e) {
-      LOGGER.error("@@@@@@  File not found at a given location @@@@@@ : " + fileName);
+      LOGGER.error("@@@@@@  File not found at a given location @@@@@@ : " + removeAKSK(fileName));
     }
     return false;
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75ec79e3/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
b/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
index 7345d54..46311db 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
@@ -21,6 +21,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
 import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.datastore.filesystem.CarbonFile
 import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.util.CarbonUtil
 import org.apache.carbondata.processing.exception.DataLoadingException
 
 object FileUtils {
@@ -38,11 +39,11 @@ object FileUtils {
       val fileName = carbonFile.getName
       if (carbonFile.getSize == 0) {
         LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-            .warn(s"skip empty input file: $path")
+            .warn(s"skip empty input file: ${CarbonUtil.removeAKSK(path)}")
       } else if (fileName.startsWith(CarbonCommonConstants.UNDERSCORE) ||
                  fileName.startsWith(CarbonCommonConstants.POINT)) {
         LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-            .warn(s"skip invisible input file: $path")
+            .warn(s"skip invisible input file: ${CarbonUtil.removeAKSK(path)}")
       } else {
         stringBuild.append(path.replace('\\', '/')).append(CarbonCommonConstants.COMMA)
       }
@@ -63,7 +64,8 @@ object FileUtils {
         val fileType = FileFactory.getFileType(filePaths(i))
         val carbonFile = FileFactory.getCarbonFile(filePaths(i), fileType)
         if (!carbonFile.exists()) {
-          throw new DataLoadingException(s"The input file does not exist: ${filePaths(i)}"
)
+          throw new DataLoadingException(
+            s"The input file does not exist: ${CarbonUtil.removeAKSK(filePaths(i))}" )
         }
         getPathsFromCarbonFile(carbonFile, stringBuild)
       }


Mime
View raw message