carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [12/14] carbondata git commit: [CARBONDATA-2017] Fix input path checking when loading data from multiple paths
Date Sat, 13 Jan 2018 08:10:41 GMT
[CARBONDATA-2017] Fix input path checking when loading data from multiple paths

This closes #1787


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6d77fdff
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6d77fdff
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6d77fdff

Branch: refs/heads/carbonstore
Commit: 6d77fdff444952a9c9ac8b02c529f9e387164be2
Parents: 913837f
Author: Manhua <kevinjmh@qq.com>
Authored: Wed Jan 10 21:22:57 2018 +0800
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Fri Jan 12 22:24:08 2018 +0800

----------------------------------------------------------------------
 .../src/main/scala/org/apache/spark/util/FileUtils.scala          | 3 ++-
 .../sql/execution/command/management/CarbonLoadDataCommand.scala  | 3 +--
 2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/6d77fdff/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
b/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
index b60d030..95ba318 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
@@ -73,7 +73,8 @@ object FileUtils {
       val stringBuild = new StringBuilder()
       val filePaths = inputPath.split(",")
       for (i <- 0 until filePaths.size) {
-        val fileType = FileFactory.getFileType(filePaths(i))
+        val filePath = CarbonUtil.checkAndAppendHDFSUrl(filePaths(i))
+        val fileType = FileFactory.getFileType(filePath)
         val carbonFile = FileFactory.getCarbonFile(filePaths(i), fileType, hadoopConf)
         if (!carbonFile.exists()) {
           throw new DataLoadingException(

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6d77fdff/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 1828557..9577615 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -147,8 +147,7 @@ case class CarbonLoadDataCommand(
       val factPath = if (dataFrame.isDefined) {
         ""
       } else {
-        FileUtils.getPaths(
-          CarbonUtil.checkAndAppendHDFSUrl(factPathFromUser), hadoopConf)
+        FileUtils.getPaths(factPathFromUser, hadoopConf)
       }
       carbonLoadModel.setFactFilePath(factPath)
       carbonLoadModel.setAggLoadRequest(


Mime
View raw message