ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject ambari git commit: AMBARI-13345. Views: HDFS API utility needs to work with WASB (Gaurav Nagar via smohanty)
Date Tue, 13 Oct 2015 13:54:11 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 600d1dc6d -> e9263e486


AMBARI-13345. Views: HDFS API utility needs to work with WASB (Gaurav Nagar via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e9263e48
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e9263e48
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e9263e48

Branch: refs/heads/branch-2.1
Commit: e9263e486a0068dba1430ea76190cb91778f0473
Parents: 600d1dc
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Tue Oct 13 06:52:33 2015 -0700
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Tue Oct 13 06:53:58 2015 -0700

----------------------------------------------------------------------
 .../view/hive/resources/files/FileService.java  |  3 ++
 contrib/views/utils/pom.xml                     |  5 +++
 .../view/utils/hdfs/ConfigurationBuilder.java   | 45 ++++++++++++++++++--
 3 files changed, 50 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/e9263e48/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
index 6718093..1b8d7cc 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
@@ -75,6 +75,9 @@ public class FileService extends BaseService {
   @Path("{filePath:.*}")
   @Produces(MediaType.APPLICATION_JSON)
   public Response getFilePage(@PathParam("filePath") String filePath, @QueryParam("page")
Long page) throws IOException, InterruptedException {
+    if (!filePath.startsWith("/") && !filePath.startsWith(".")) {
+      filePath = "/" + filePath;  // some servers strip double slashes in URL
+    }
     LOG.debug("Reading file " + filePath);
     try {
       FileResource file = new FileResource();

http://git-wip-us.apache.org/repos/asf/ambari/blob/e9263e48/contrib/views/utils/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/views/utils/pom.xml b/contrib/views/utils/pom.xml
index dee0a7c..c28c380 100644
--- a/contrib/views/utils/pom.xml
+++ b/contrib/views/utils/pom.xml
@@ -133,6 +133,11 @@
       <artifactId>commons-io</artifactId>
       <version>2.4</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-azure</artifactId>
+      <version>${hadoop-version}</version>
+    </dependency>
   </dependencies>
 
   <properties>

http://git-wip-us.apache.org/repos/asf/ambari/blob/e9263e48/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java
----------------------------------------------------------------------
diff --git a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java
b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java
index d992c2e..a739c84 100644
--- a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java
+++ b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java
@@ -23,6 +23,8 @@ import org.apache.ambari.view.utils.ambari.AmbariApi;
 import org.apache.ambari.view.utils.ambari.NoClusterAssociatedException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
+import org.apache.hadoop.fs.azure.Wasb;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
 import org.slf4j.Logger;
@@ -66,7 +68,7 @@ public class ConfigurationBuilder {
   private AmbariApi ambariApi = null;
   private AuthConfigurationBuilder authParamsBuilder;
   private Map<String, String> authParams;
-
+  private URI defaultFsUri;
   /**
    * Constructor of ConfigurationBuilder based on ViewContext
    * @param context ViewContext
@@ -87,12 +89,14 @@ public class ConfigurationBuilder {
 
         LOG.info("HA HDFS cluster found.");
       } else {
-        if (!hasPort(defaultFS)) {
+        if (defaultFS.startsWith("hdfs://") && !hasPort(defaultFS)) {
           defaultFS = addPortIfMissing(defaultFS);
         }
       }
 
-      } catch (URISyntaxException e) {
+      defaultFsUri = new URI(defaultFS);
+
+    } catch (URISyntaxException e) {
       throw new HdfsApiException("HDFS060 Invalid " + DEFAULT_FS_INSTANCE_PROPERTY +
           "='" + defaultFS + "' URI", e);
     }
@@ -121,6 +125,21 @@ public class ConfigurationBuilder {
     return value;
   }
 
+  private void copyPropertyIfExists(String type, String key) {
+    String value;
+    try {
+      value = ambariApi.getCluster().getConfigurationValue(type, key);
+      if (value != null) {
+        conf.set(key, value);
+        LOG.debug("set " + key + " = " + value);
+      } else {
+        LOG.debug("No such property " + type + "/" + key);
+      }
+    } catch (NoClusterAssociatedException e) {
+      LOG.debug("No such property " + type + "/" + key);
+    }
+  }
+
   private void copyHAProperties(String defaultFS) throws URISyntaxException, HdfsApiException
{
     URI uri = new URI(defaultFS);
     String nameservice = uri.getHost();
@@ -152,6 +171,7 @@ public class ConfigurationBuilder {
   private String copyClusterProperty(String propertyName, String instancePropertyName) {
     String value = getProperty(HDFS_SITE, propertyName, instancePropertyName);
     conf.set(propertyName, value);
+    LOG.debug("set " + propertyName + " = " + value);
     return value;
   }
 
@@ -208,10 +228,29 @@ public class ConfigurationBuilder {
     conf.set("fs.webhdfs.impl", WebHdfsFileSystem.class.getName());
     conf.set("fs.file.impl", LocalFileSystem.class.getName());
 
+    configureWASB();
+
     return conf;
   }
 
   /**
+   * Fill Azure Blob Storage properties if wasb:// scheme configured
+   */
+  public void configureWASB() {
+    LOG.debug("defaultFsUri.getScheme() == " + defaultFsUri.getScheme());
+    if (defaultFsUri.getScheme().equals("wasb")) {
+      conf.set("fs.AbstractFileSystem.wasb.impl", Wasb.class.getName());
+      conf.set("fs.wasb.impl", NativeAzureFileSystem.class.getName());
+
+      String account = defaultFsUri.getHost();
+      LOG.debug("WASB account == " + account);
+      copyPropertyIfExists(CORE_SITE, "fs.azure.account.key." + account);
+      copyPropertyIfExists(CORE_SITE, "fs.azure.account.keyprovider." + account);
+      copyPropertyIfExists(CORE_SITE, "fs.azure.shellkeyprovider.script");
+    }
+  }
+
+  /**
    * Builds the authentication configuration
    * @return map of HDFS auth params for view
    * @throws HdfsApiException


Mime
View raw message