carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [1/2] incubator-carbondata git commit: Fixed the query fails if big data is loaded to local store.
Date Fri, 22 Jul 2016 09:05:23 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 56a1e402f -> 06e8e95cc


Fixed the query fails if big data is loaded to local store.


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/217b5afe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/217b5afe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/217b5afe

Branch: refs/heads/master
Commit: 217b5afe65450bb896bfa82521176096ecaacc82
Parents: 56a1e40
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Fri Jul 22 09:32:06 2016 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Fri Jul 22 09:32:06 2016 +0530

----------------------------------------------------------------------
 .../org/carbondata/hadoop/CarbonInputFormat.java     | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/217b5afe/hadoop/src/main/java/org/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/carbondata/hadoop/CarbonInputFormat.java
index 1cd44c9..bbf40ba 100644
--- a/hadoop/src/main/java/org/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/carbondata/hadoop/CarbonInputFormat.java
@@ -75,6 +75,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
@@ -611,6 +612,20 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void,
T> {
     return result;
   }
 
+  @Override protected boolean isSplitable(JobContext context, Path filename) {
+    try {
+      // Don't split the file if it is local file system
+      FileSystem fileSystem = filename.getFileSystem(context.getConfiguration());
+      if (fileSystem instanceof LocalFileSystem)
+      {
+        return false;
+      }
+    } catch (Exception e) {
+      return true;
+    }
+    return true;
+  }
+
   private void getFileStatusOfSegments(JobContext job, String[] segmentsToConsider,
       List<FileStatus> result) throws IOException {
     String[] partitionsToConsider = getValidPartitions(job);


Mime
View raw message