From common-commits-return-96393-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Thu Sep 5 12:52:20 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id 5C1FE180656 for ; Thu, 5 Sep 2019 14:52:20 +0200 (CEST) Received: (qmail 43489 invoked by uid 500); 5 Sep 2019 21:00:09 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 43437 invoked by uid 99); 5 Sep 2019 21:00:09 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 05 Sep 2019 21:00:09 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 77191807C8; Thu, 5 Sep 2019 12:52:18 +0000 (UTC) Date: Thu, 05 Sep 2019 12:52:18 +0000 To: "common-commits@hadoop.apache.org" Subject: [hadoop] 02/05: YARN-7537. Add ability to load hbase config from distributed file system. Contributed by Prabhu Joseph MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit From: rohithsharmaks@apache.org In-Reply-To: <156768793651.14877.14738803098590500506@gitbox.apache.org> References: <156768793651.14877.14738803098590500506@gitbox.apache.org> X-Git-Host: gitbox.apache.org X-Git-Repo: hadoop X-Git-Refname: refs/heads/branch-3.2 X-Git-Reftype: branch X-Git-Rev: 6110af2d1df75d37a2d4e4095e132df23c4ed429 X-Git-NotificationType: diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated Message-Id: <20190905125218.77191807C8@gitbox.apache.org> This is an automated email from the ASF dual-hosted git repository. rohithsharmaks pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit 6110af2d1df75d37a2d4e4095e132df23c4ed429 Author: Eric Yang AuthorDate: Tue Jun 4 19:26:06 2019 -0400 YARN-7537. Add ability to load hbase config from distributed file system. Contributed by Prabhu Joseph --- .../pom.xml | 13 ++++ .../storage/common/HBaseTimelineStorageUtils.java | 32 ++++++---- .../common/TestHBaseTimelineStorageUtils.java | 74 ++++++++++++++++++++++ 3 files changed, 106 insertions(+), 13 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml index ff7fb6e..74af322 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml @@ -76,6 +76,19 @@ org.apache.hadoop + hadoop-hdfs + test + + + + org.apache.hadoop + hadoop-hdfs + test-jar + test + + + + org.apache.hadoop hadoop-yarn-api provided diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStor [...] index f4cd6fb..93feb82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java @@ -17,11 +17,13 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.common; -import java.net.MalformedURLException; -import java.net.URL; +import java.io.IOException; import java.util.Arrays; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.Query; @@ -40,7 +42,6 @@ public final class HBaseTimelineStorageUtils { private HBaseTimelineStorageUtils() { } - /** * @param conf YARN configuration. Used to see if there is an explicit config * pointing to the HBase config file to read. It should not be null @@ -48,28 +49,33 @@ public final class HBaseTimelineStorageUtils { * @return a configuration with the HBase configuration from the classpath, * optionally overwritten by the timeline service configuration URL if * specified. - * @throws MalformedURLException if a timeline service HBase configuration URL - * is specified but is a malformed URL. + * @throws IOException if a timeline service HBase configuration URL + * is specified but unable to read it. */ public static Configuration getTimelineServiceHBaseConf(Configuration conf) - throws MalformedURLException { + throws IOException { if (conf == null) { throw new NullPointerException(); } Configuration hbaseConf; - String timelineServiceHBaseConfFileURL = + String timelineServiceHBaseConfFilePath = conf.get(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE); - if (timelineServiceHBaseConfFileURL != null - && timelineServiceHBaseConfFileURL.length() > 0) { + + if (timelineServiceHBaseConfFilePath != null + && timelineServiceHBaseConfFilePath.length() > 0) { LOG.info("Using hbase configuration at " + - timelineServiceHBaseConfFileURL); + timelineServiceHBaseConfFilePath); // create a clone so that we don't mess with out input one hbaseConf = new Configuration(conf); Configuration plainHBaseConf = new Configuration(false); - URL hbaseSiteXML = new URL(timelineServiceHBaseConfFileURL); - plainHBaseConf.addResource(hbaseSiteXML); - HBaseConfiguration.merge(hbaseConf, plainHBaseConf); + Path hbaseConfigPath = new Path(timelineServiceHBaseConfFilePath); + try (FileSystem fs = + FileSystem.newInstance(hbaseConfigPath.toUri(), conf); + FSDataInputStream in = fs.open(hbaseConfigPath)) { + plainHBaseConf.addResource(in); + HBaseConfiguration.merge(hbaseConf, plainHBaseConf); + } } else { // default to what is on the classpath hbaseConf = HBaseConfiguration.create(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTime [...] index 402a89b..46bb8ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java @@ -18,16 +18,90 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.common; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + /** * Unit tests for HBaseTimelineStorageUtils static methos. */ public class TestHBaseTimelineStorageUtils { + private String hbaseConfigPath = "target/hbase-site.xml"; + + @Before + public void setup() throws IOException { + // Input Hbase Configuration + Configuration hbaseConf = new Configuration(); + hbaseConf.set("input", "test"); + + //write the document to a buffer (not directly to the file, as that + //can cause the file being written to get read which will then fail. + ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); + hbaseConf.writeXml(bytesOut); + bytesOut.close(); + + //write the bytes to the file + File file = new File(hbaseConfigPath); + OutputStream os = new FileOutputStream(file); + os.write(bytesOut.toByteArray()); + os.close(); + } + @Test(expected=NullPointerException.class) public void testGetTimelineServiceHBaseConfNullArgument() throws Exception { HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(null); } + @Test + public void testWithHbaseConfAtLocalFileSystem() throws IOException { + // Verifying With Hbase Conf from Local FileSystem + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE, + hbaseConfigPath); + Configuration hbaseConfFromLocal = + HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(conf); + Assert.assertEquals("Failed to read hbase config from Local FileSystem", + "test", hbaseConfFromLocal.get("input")); + } + + @Test + public void testWithHbaseConfAtHdfsFileSystem() throws IOException { + MiniDFSCluster hdfsCluster = null; + try { + HdfsConfiguration hdfsConfig = new HdfsConfiguration(); + hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig) + .numDataNodes(1).build(); + + FileSystem fs = hdfsCluster.getFileSystem(); + Path path = new Path("/tmp/hdfs-site.xml"); + fs.copyFromLocalFile(new Path(hbaseConfigPath), path); + + // Verifying With Hbase Conf from HDFS FileSystem + Configuration conf = new Configuration(hdfsConfig); + conf.set(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE, + path.toString()); + Configuration hbaseConfFromHdfs = + HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(conf); + Assert.assertEquals("Failed to read hbase config from Hdfs FileSystem", + "test", hbaseConfFromHdfs.get("input")); + } finally { + if (hdfsCluster != null) { + hdfsCluster.shutdown(); + } + } + } + } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org