hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rohithsharm...@apache.org
Subject [hadoop] 02/05: YARN-7537. Add ability to load hbase config from distributed file system. Contributed by Prabhu Joseph
Date Thu, 05 Sep 2019 12:52:18 GMT
This is an automated email from the ASF dual-hosted git repository.

rohithsharmaks pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit 6110af2d1df75d37a2d4e4095e132df23c4ed429
Author: Eric Yang <eyang@apache.org>
AuthorDate: Tue Jun 4 19:26:06 2019 -0400

    YARN-7537.  Add ability to load hbase config from distributed file system.
                Contributed by Prabhu Joseph
---
 .../pom.xml                                        | 13 ++++
 .../storage/common/HBaseTimelineStorageUtils.java  | 32 ++++++----
 .../common/TestHBaseTimelineStorageUtils.java      | 74 ++++++++++++++++++++++
 3 files changed, 106 insertions(+), 13 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
index ff7fb6e..74af322 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
@@ -76,6 +76,19 @@
 
     <dependency>
       <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-api</artifactId>
       <scope>provided</scope>
     </dependency>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStor
[...]
index f4cd6fb..93feb82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
@@ -17,11 +17,13 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.storage.common;
 
-import java.net.MalformedURLException;
-import java.net.URL;
+import java.io.IOException;
 import java.util.Arrays;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.Query;
@@ -40,7 +42,6 @@ public final class HBaseTimelineStorageUtils {
   private HBaseTimelineStorageUtils() {
   }
 
-
   /**
    * @param conf YARN configuration. Used to see if there is an explicit config
    *          pointing to the HBase config file to read. It should not be null
@@ -48,28 +49,33 @@ public final class HBaseTimelineStorageUtils {
    * @return a configuration with the HBase configuration from the classpath,
    *         optionally overwritten by the timeline service configuration URL if
    *         specified.
-   * @throws MalformedURLException if a timeline service HBase configuration URL
-   *           is specified but is a malformed URL.
+   * @throws IOException if a timeline service HBase configuration URL
+   *           is specified but unable to read it.
    */
   public static Configuration getTimelineServiceHBaseConf(Configuration conf)
-      throws MalformedURLException {
+      throws IOException {
     if (conf == null) {
       throw new NullPointerException();
     }
 
     Configuration hbaseConf;
-    String timelineServiceHBaseConfFileURL =
+    String timelineServiceHBaseConfFilePath =
         conf.get(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE);
-    if (timelineServiceHBaseConfFileURL != null
-        && timelineServiceHBaseConfFileURL.length() > 0) {
+
+    if (timelineServiceHBaseConfFilePath != null
+          && timelineServiceHBaseConfFilePath.length() > 0) {
       LOG.info("Using hbase configuration at " +
-          timelineServiceHBaseConfFileURL);
+          timelineServiceHBaseConfFilePath);
       // create a clone so that we don't mess with out input one
       hbaseConf = new Configuration(conf);
       Configuration plainHBaseConf = new Configuration(false);
-      URL hbaseSiteXML = new URL(timelineServiceHBaseConfFileURL);
-      plainHBaseConf.addResource(hbaseSiteXML);
-      HBaseConfiguration.merge(hbaseConf, plainHBaseConf);
+      Path hbaseConfigPath = new Path(timelineServiceHBaseConfFilePath);
+      try (FileSystem fs =
+          FileSystem.newInstance(hbaseConfigPath.toUri(), conf);
+          FSDataInputStream in = fs.open(hbaseConfigPath)) {
+        plainHBaseConf.addResource(in);
+        HBaseConfiguration.merge(hbaseConf, plainHBaseConf);
+      }
     } else {
       // default to what is on the classpath
       hbaseConf = HBaseConfiguration.create(conf);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTime
[...]
index 402a89b..46bb8ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
@@ -18,16 +18,90 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.storage.common;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
 /**
  * Unit tests for HBaseTimelineStorageUtils static methos.
  */
 public class TestHBaseTimelineStorageUtils {
 
+  private String hbaseConfigPath = "target/hbase-site.xml";
+
+  @Before
+  public void setup() throws IOException {
+    // Input Hbase Configuration
+    Configuration hbaseConf = new Configuration();
+    hbaseConf.set("input", "test");
+
+    //write the document to a buffer (not directly to the file, as that
+    //can cause the file being written to get read which will then fail.
+    ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+    hbaseConf.writeXml(bytesOut);
+    bytesOut.close();
+
+    //write the bytes to the file
+    File file = new File(hbaseConfigPath);
+    OutputStream os = new FileOutputStream(file);
+    os.write(bytesOut.toByteArray());
+    os.close();
+  }
+
   @Test(expected=NullPointerException.class)
   public void testGetTimelineServiceHBaseConfNullArgument() throws Exception {
     HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(null);
   }
 
+  @Test
+  public void testWithHbaseConfAtLocalFileSystem() throws IOException {
+    // Verifying With Hbase Conf from Local FileSystem
+    Configuration conf = new Configuration();
+    conf.set(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE,
+        hbaseConfigPath);
+    Configuration hbaseConfFromLocal =
+        HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(conf);
+    Assert.assertEquals("Failed to read hbase config from Local FileSystem",
+        "test", hbaseConfFromLocal.get("input"));
+  }
+
+  @Test
+  public void testWithHbaseConfAtHdfsFileSystem() throws IOException {
+    MiniDFSCluster hdfsCluster = null;
+    try {
+      HdfsConfiguration hdfsConfig = new HdfsConfiguration();
+      hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig)
+          .numDataNodes(1).build();
+
+      FileSystem fs = hdfsCluster.getFileSystem();
+      Path path = new Path("/tmp/hdfs-site.xml");
+      fs.copyFromLocalFile(new Path(hbaseConfigPath), path);
+
+      // Verifying With Hbase Conf from HDFS FileSystem
+      Configuration conf = new Configuration(hdfsConfig);
+      conf.set(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE,
+          path.toString());
+      Configuration hbaseConfFromHdfs =
+          HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(conf);
+      Assert.assertEquals("Failed to read hbase config from Hdfs FileSystem",
+          "test", hbaseConfFromHdfs.get("input"));
+    } finally {
+      if (hdfsCluster != null) {
+        hdfsCluster.shutdown();
+      }
+    }
+  }
+
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message