drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ve...@apache.org
Subject [7/7] drill git commit: DRILL-4383: Allow custom configurations to be specified for a FileSystem plugin
Date Wed, 02 Mar 2016 23:52:26 GMT
DRILL-4383: Allow custom configurations to be specified for a FileSystem plugin

add an example s3 plugin, disabled by default

Closes #375


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/0842851c
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/0842851c
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/0842851c

Branch: refs/heads/master
Commit: 0842851c854595f140779e9ed09331dbb63f6623
Parents: 2ffe311
Author: Jason Altekruse <altekrusejason@gmail.com>
Authored: Thu Feb 11 16:52:38 2016 -0800
Committer: vkorukanti <venki.korukanti@gmail.com>
Committed: Wed Mar 2 14:08:51 2016 -0800

----------------------------------------------------------------------
 .../drill/exec/store/dfs/FileSystemConfig.java  | 10 ++++
 .../drill/exec/store/dfs/FileSystemPlugin.java  |  5 ++
 .../resources/bootstrap-storage-plugins.json    | 56 ++++++++++++++++++++
 3 files changed, 71 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/0842851c/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
index d88750f..adc3909 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.store.dfs;
 
 import java.util.Map;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import org.apache.drill.common.logical.FormatPluginConfig;
 import org.apache.drill.common.logical.StoragePluginConfig;
 
@@ -29,6 +30,7 @@ public class FileSystemConfig extends StoragePluginConfig {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemConfig.class);
   public static final String NAME = "file";
   public String connection;
+  public Map<String, String> config;
   public Map<String, WorkspaceConfig> workspaces;
   public Map<String, FormatPluginConfig> formats;
 
@@ -36,6 +38,7 @@ public class FileSystemConfig extends StoragePluginConfig {
   public int hashCode() {
     final int prime = 31;
     int result = 1;
+    result = prime * result + ((config == null) ? 0 : config.hashCode());
     result = prime * result + ((connection == null) ? 0 : connection.hashCode());
     result = prime * result + ((formats == null) ? 0 : formats.hashCode());
     result = prime * result + ((workspaces == null) ? 0 : workspaces.hashCode());
@@ -75,6 +78,13 @@ public class FileSystemConfig extends StoragePluginConfig {
     } else if (!workspaces.equals(other.workspaces)) {
       return false;
     }
+    if (config == null) {
+      if (other.config != null) {
+        return false;
+      }
+    } else if (!config.equals(other.config)) {
+      return false;
+    }
     return true;
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/0842851c/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
index 6a9bddb..7f2a9c1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
@@ -69,6 +69,11 @@ public class FileSystemPlugin extends AbstractStoragePlugin{
     try {
 
       fsConf = new Configuration();
+      if (config.config != null) {
+        for (String s : config.config.keySet()) {
+          fsConf.set(s, config.config.get(s));
+        }
+      }
       fsConf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.connection);
       fsConf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
       fsConf.set("fs.drill-local.impl", LocalSyncableFileSystem.class.getName());

http://git-wip-us.apache.org/repos/asf/drill/blob/0842851c/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
index f5d3880..13d29ea 100644
--- a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
+++ b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
@@ -51,6 +51,62 @@
         }
       }
     },
+    s3: {
+      type: "file",
+      connection: "s3a://my.bucket.location.com",
+      enabled : false,
+      config : {
+        "fs.s3a.access.key": "ID",
+        "fs.s3a.secret.key": "SECRET"
+      },
+      workspaces: {
+        "root" : {
+          location: "/",
+          writable: false
+        },
+        "tmp" : {
+          location: "/tmp",
+          writable: true
+        }
+      },
+      formats: {
+        "psv" : {
+          type: "text",
+          extensions: [ "tbl" ],
+          delimiter: "|"
+        },
+        "csv" : {
+          type: "text",
+          extensions: [ "csv" ],
+          delimiter: ","
+        },
+        "tsv" : {
+          type: "text",
+          extensions: [ "tsv" ],
+          delimiter: "\t"
+        },
+        "parquet" : {
+          type: "parquet"
+        },
+        "json" : {
+          type: "json",
+          extensions: [ "json" ]
+        },
+        "avro" : {
+          type: "avro"
+        },
+        "sequencefile": {
+          type : "sequencefile",
+          extensions: [ "seq" ]
+        },
+        "csvh" : {
+          type: "text",
+          extensions: [ "csvh" ],
+          delimiter: ",",
+          extractHeader: true
+        }
+      }
+    },
 
     cp: {
       type: "file",


Mime
View raw message