From common-commits-return-83532-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Thu May 31 17:49:58 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id AA606180632 for ; Thu, 31 May 2018 17:49:57 +0200 (CEST) Received: (qmail 91925 invoked by uid 500); 31 May 2018 15:49:55 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 91727 invoked by uid 99); 31 May 2018 15:49:55 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 31 May 2018 15:49:55 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 6FCE0E10BD; Thu, 31 May 2018 15:49:55 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xyao@apache.org To: common-commits@hadoop.apache.org Date: Thu, 31 May 2018 15:49:58 -0000 Message-Id: <525eed3aff74422e9d003567909aa805@git.apache.org> In-Reply-To: <1b35a5e8d37342629b07655527f0a644@git.apache.org> References: <1b35a5e8d37342629b07655527f0a644@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [04/50] [abbrv] hadoop git commit: HDDS-96. Add an option in ozone script to generate a site file with minimally required ozone configs. Contributed by Dinesh Chitlangia. HDDS-96. Add an option in ozone script to generate a site file with minimally required ozone configs. Contributed by Dinesh Chitlangia. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8733012a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8733012a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8733012a Branch: refs/heads/HDDS-4 Commit: 8733012ae35f2762d704f94975a762885d116795 Parents: 1e0d4b1 Author: Anu Engineer Authored: Fri May 25 13:06:14 2018 -0700 Committer: Anu Engineer Committed: Fri May 25 13:06:14 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hdds/conf/OzoneConfiguration.java | 6 +- hadoop-ozone/common/src/main/bin/ozone | 4 + ...TestGenerateOzoneRequiredConfigurations.java | 100 +++++++++++ .../GenerateOzoneRequiredConfigurations.java | 174 +++++++++++++++++++ .../hadoop/ozone/genconf/package-info.java | 24 +++ 5 files changed, 305 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8733012a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index f07718c..36d953c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -137,7 +137,7 @@ public class OzoneConfiguration extends Configuration { @Override public String toString() { - return this.getName() + " " + this.getValue() + this.getTag(); + return this.getName() + " " + this.getValue() + " " + this.getTag(); } @Override @@ -152,11 +152,11 @@ public class OzoneConfiguration extends Configuration { } } - public static void activate(){ + public static void activate() { // adds the default resources Configuration.addDefaultResource("hdfs-default.xml"); Configuration.addDefaultResource("hdfs-site.xml"); Configuration.addDefaultResource("ozone-default.xml"); Configuration.addDefaultResource("ozone-site.xml"); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8733012a/hadoop-ozone/common/src/main/bin/ozone ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone index 00261c7..6843bdd 100755 --- a/hadoop-ozone/common/src/main/bin/ozone +++ b/hadoop-ozone/common/src/main/bin/ozone @@ -47,6 +47,7 @@ function hadoop_usage hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service" hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager " hadoop_add_subcommand "version" client "print the version" + hadoop_add_subcommand "genconf" client "generate minimally required ozone configs and output to ozone-site.xml in specified path" hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false } @@ -118,6 +119,9 @@ function ozonecmd_case version) HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo ;; + genconf) + HADOOP_CLASSNAME=org.apache.hadoop.ozone.genconf.GenerateOzoneRequiredConfigurations + ;; *) HADOOP_CLASSNAME="${subcmd}" if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then http://git-wip-us.apache.org/repos/asf/hadoop/blob/8733012a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java new file mode 100644 index 0000000..82582a6 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.genconf; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + + +/** + * Tests GenerateOzoneRequiredConfigurations. + */ +public class TestGenerateOzoneRequiredConfigurations { + private static MiniOzoneCluster cluster; + private static OzoneConfiguration conf; + + /** + * Create a MiniDFSCluster for testing. + *

+ * Ozone is made active by setting OZONE_ENABLED = true and + * OZONE_HANDLER_TYPE_KEY = "distributed" + * + * @throws IOException + */ + @BeforeClass + public static void init() throws Exception { + conf = new OzoneConfiguration(); + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build(); + cluster.waitForClusterToBeReady(); + } + + /** + * Shutdown MiniDFSCluster. + */ + @AfterClass + public static void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + /** + * Tests a valid path and generates ozone-site.xml. + * @throws Exception + */ + @Test + public void generateConfigurationsSuccess() throws Exception { + String[] args = new String[]{"-output", "."}; + GenerateOzoneRequiredConfigurations.main(args); + + Assert.assertEquals("Path is valid", + true, GenerateOzoneRequiredConfigurations.isValidPath(args[1])); + + Assert.assertEquals("Permission is valid", + true, GenerateOzoneRequiredConfigurations.canWrite(args[1])); + + Assert.assertEquals("Config file generated", + 0, GenerateOzoneRequiredConfigurations.generateConfigurations(args[1])); + } + + /** + * Test to avoid generating ozone-site.xml when invalid permission. + * @throws Exception + */ + @Test + public void generateConfigurationsFailure() throws Exception { + String[] args = new String[]{"-output", "/"}; + GenerateOzoneRequiredConfigurations.main(args); + + Assert.assertEquals("Path is valid", + true, GenerateOzoneRequiredConfigurations.isValidPath(args[1])); + + Assert.assertEquals("Invalid permission", + false, GenerateOzoneRequiredConfigurations.canWrite(args[1])); + + Assert.assertEquals("Config file not generated", + 1, GenerateOzoneRequiredConfigurations.generateConfigurations(args[1])); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8733012a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java new file mode 100644 index 0000000..6296c9d --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.genconf; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import java.io.File; +import java.net.URL; +import java.nio.file.InvalidPathException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +/** + * GenerateOzoneRequiredConfigurations - A tool to generate ozone-site.xml
+ * This tool generates an ozone-site.xml with minimally required configs. + * This tool can be invoked as follows:
+ *

    + *
  • ozone genconf -output
    • + *
  • ozone genconf -help
    • + *
+ */ +public final class GenerateOzoneRequiredConfigurations { + + private static final String OUTPUT = "-output"; + private static final String HELP = "-help"; + private static final String USAGE = "Usage: \nozone genconf " + + OUTPUT + " \n" + + "ozone genconf " + + HELP; + private static final int SUCCESS = 0; + private static final int FAILURE = 1; + + private GenerateOzoneRequiredConfigurations() { + + } + /** + * Entry point for using genconf tool. + * + * @param args + * @throws JAXBException + */ + public static void main(String[] args) { + + try { + if (args.length == 0) { + System.out.println(USAGE); + System.exit(1); + } + + switch (args[0]) { + case OUTPUT: + if (args.length > 1) { + int result = generateConfigurations(args[1]); + } else { + System.out.println("Path to output file is mandatory"); + System.out.println(USAGE); + System.exit(1); + } + break; + + case HELP: + System.out.println(USAGE); + System.exit(0); + break; + + default: + System.out.println(USAGE); + System.exit(1); + } + + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * Check if the path is valid. + * + * @param path + * @return true, if path is valid, else return false + */ + public static boolean isValidPath(String path) { + try { + Paths.get(path); + } catch (InvalidPathException | NullPointerException ex) { + return false; + } + return true; + } + + /** + * Check if user has permission to write in the specified path. + * + * @param path + * @return true, if the user has permission to write, else returns false + */ + public static boolean canWrite(String path) { + File file = new File(path); + return file.canWrite(); + } + + /** + * Generate ozone-site.xml at specified path. + * + * @param path + * @return SUCCESS(0) if file can be generated, else returns FAILURE(1) + * @throws JAXBException + */ + public static int generateConfigurations(String path) throws JAXBException { + + if (!isValidPath(path)) { + System.out.println("Invalid path or insufficient permission"); + return FAILURE; + } + + if (!canWrite(path)) { + System.out.println("Invalid path or insufficient permission"); + return FAILURE; + } + + OzoneConfiguration oc = new OzoneConfiguration(); + + ClassLoader cL = Thread.currentThread().getContextClassLoader(); + if (cL == null) { + cL = OzoneConfiguration.class.getClassLoader(); + } + URL url = cL.getResource("ozone-default.xml"); + + List allProperties = + oc.readPropertyFromXml(url); + + List requiredProperties = new ArrayList<>(); + + for (OzoneConfiguration.Property p : allProperties) { + if (p.getTag() != null && p.getTag().contains("REQUIRED")) { + requiredProperties.add(p); + } + } + + OzoneConfiguration.XMLConfiguration requiredConfig = + new OzoneConfiguration.XMLConfiguration(); + requiredConfig.setProperties(requiredProperties); + + JAXBContext context = + JAXBContext.newInstance(OzoneConfiguration.XMLConfiguration.class); + Marshaller m = context.createMarshaller(); + m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + m.marshal(requiredConfig, new File(path, "ozone-site.xml")); + + System.out.println("ozone-site.xml has been generated at " + path); + + return SUCCESS; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8733012a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/package-info.java new file mode 100644 index 0000000..4817d39 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/package-info.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.genconf; + +/** + * Command line tool to generate required Ozone configs to an ozone-site.xml. + */ + --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org