drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject [2/8] drill git commit: DRILL-1545: Allow custom extensions for json files
Date Thu, 07 May 2015 18:02:34 GMT
DRILL-1545: Allow custom extensions for json files

Fix case where compressed files are skipped.

Add jackson annotation to avoid serializing the default extensions list to enable forwards
compatibility.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/efb4e701
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/efb4e701
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/efb4e701

Branch: refs/heads/master
Commit: efb4e70152b2ccca3bc9111466cf9406373ee67e
Parents: b0ac3fb
Author: Paul Pearcy <ppearcy@gmail.com>
Authored: Mon Jan 26 00:56:04 2015 -0500
Committer: Jason Altekruse <altekrusejason@gmail.com>
Committed: Thu May 7 10:06:23 2015 -0700

----------------------------------------------------------------------
 .../exec/store/dfs/BasicFormatMatcher.java      | 13 ++++++----
 .../exec/store/easy/json/JSONFormatPlugin.java  | 26 ++++++++++++++------
 .../resources/bootstrap-storage-plugins.json    |  6 +++--
 3 files changed, 31 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/efb4e701/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
index 3768aea..1cb9f82 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
@@ -89,17 +89,20 @@ public class BasicFormatMatcher extends FormatMatcher{
     if (compressible) {
       codec = codecFactory.getCodec(status.getPath());
     }
-    String fileName;
+    String fileName = status.getPath().toString();
+    String fileNameHacked = null;
     if (codec != null) {
-      String path = status.getPath().toString();
-      fileName = path.substring(0, path.lastIndexOf('.'));
-    } else {
-      fileName = status.getPath().toString();
+        fileNameHacked = fileName.substring(0, fileName.lastIndexOf('.'));
     }
+
+    // Check for a matching pattern for compressed and uncompressed file name
     for (Pattern p : patterns) {
       if (p.matcher(fileName).matches()) {
         return true;
       }
+      if (fileNameHacked != null  &&  p.matcher(fileNameHacked).matches()) {
+        return true;
+      }
     }
 
     if (matcher.matches(fs, status)) {

http://git-wip-us.apache.org/repos/asf/drill/blob/efb4e701/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
index 32e34b4..8686b26 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
@@ -21,7 +21,9 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.logical.FormatPluginConfig;
@@ -45,20 +47,18 @@ import org.apache.hadoop.fs.FileSystem;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.collect.Maps;
 
-public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> {
+public class
+        JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> {
 
   private static final boolean IS_COMPRESSIBLE = true;
   private static final String DEFAULT_NAME = "json";
-  private static final List<String> DEFAULT_EXTS = ImmutableList.of("json");
 
   public JSONFormatPlugin(String name, DrillbitContext context, Configuration fsConf, StoragePluginConfig
storageConfig) {
     this(name, context, fsConf, storageConfig, new JSONFormatConfig());
   }
 
-  public JSONFormatPlugin(String name, DrillbitContext context, Configuration fsConf, StoragePluginConfig
config,
-      JSONFormatConfig formatPluginConfig) {
-    super(name, context, fsConf, config, formatPluginConfig, true, false, false, IS_COMPRESSIBLE,
-        DEFAULT_EXTS, DEFAULT_NAME);
+  public JSONFormatPlugin(String name, DrillbitContext context, Configuration fsConf, StoragePluginConfig
config, JSONFormatConfig formatPluginConfig) {
+    super(name, context, fsConf, config, formatPluginConfig, true, false, false, IS_COMPRESSIBLE,
formatPluginConfig.getExtensions(), DEFAULT_NAME);
   }
 
   @Override
@@ -92,6 +92,18 @@ public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig>
{
   @JsonTypeName("json")
   public static class JSONFormatConfig implements FormatPluginConfig {
 
+    public List<String> extensions;
+    private static final List<String> DEFAULT_EXTS = ImmutableList.of("json");
+
+    @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+    public List<String> getExtensions() {
+      if (extensions == null) {
+        // when loading an old JSONFormatConfig that doesn't contain an "extensions" attribute
+        return DEFAULT_EXTS;
+      }
+      return extensions;
+    }
+
     @Override
     public int hashCode() {
       return 31;
@@ -106,9 +118,9 @@ public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig>
{
       } else if (getClass() == obj.getClass()) {
         return true;
       }
+
       return false;
     }
-
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/efb4e701/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
index 3253e80..1801f1c 100644
--- a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
+++ b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
@@ -33,7 +33,8 @@
           type: "parquet"
         },
         "json" : {
-          type: "json"
+          type: "json",
+          extensions: [ "json" ]
         },
         "avro" : {
           type: "avro"
@@ -56,7 +57,8 @@
                   delimiter: "\t"
         },
         "json" : {
-          type: "json"
+          type: "json",
+          extensions: [ "json" ]
         },
         "parquet" : {
           type: "parquet"


Mime
View raw message