Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 50115200D17 for ; Wed, 30 Aug 2017 18:39:38 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 4EC1E169822; Wed, 30 Aug 2017 16:39:38 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 1620116980B for ; Wed, 30 Aug 2017 18:39:33 +0200 (CEST) Received: (qmail 6528 invoked by uid 500); 30 Aug 2017 16:39:33 -0000 Mailing-List: contact commits-help@oodt.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@oodt.apache.org Delivered-To: mailing list commits@oodt.apache.org Received: (qmail 4543 invoked by uid 99); 30 Aug 2017 16:39:31 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 30 Aug 2017 16:39:31 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BF042F555B; Wed, 30 Aug 2017 16:39:31 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mattmann@apache.org To: commits@oodt.apache.org Date: Wed, 30 Aug 2017 16:40:11 -0000 Message-Id: In-Reply-To: <602f1c7b278843318b280f88fc5fbb83@git.apache.org> References: <602f1c7b278843318b280f88fc5fbb83@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [42/52] [abbrv] oodt git commit: Added missing conf-publisher binary and minor improvements archived-at: Wed, 30 Aug 2017 16:39:38 -0000 Added missing conf-publisher binary and minor improvements Project: http://git-wip-us.apache.org/repos/asf/oodt/repo Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/39b6c888 Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/39b6c888 Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/39b6c888 Branch: refs/heads/master Commit: 39b6c888512b74feaa9d5494c7bd7268f4b42f6e Parents: 8b02792 Author: Imesha Sudasingha Authored: Wed Jul 26 21:51:41 2017 +0530 Committer: Imesha Sudasingha Committed: Fri Jul 28 21:20:43 2017 +0530 ---------------------------------------------------------------------- config/pom.xml | 36 +- config/src/main/assembly/assembly.xml | 4 +- .../DistributedConfigurationManager.java | 1 - .../DistributedConfigurationPublisher.java | 257 ++++++++++++++ .../config/distributed/cli/ConfigPublisher.java | 105 ++++++ .../cli/DistributedConfigurationPublisher.java | 331 ------------------- .../config/distributed/utils/FilePathUtils.java | 6 +- .../src/main/resources/etc/config-publisher.xml | 16 +- config/src/main/resources/etc/log4j.xml | 5 +- config/src/main/scripts/conf-publisher | 36 ++ .../DistributedConfigurationManagerTest.java | 25 +- .../DistributedConfigurationPublisherTest.java | 6 +- .../src/test/resources/etc/config-publisher.xml | 4 +- 13 files changed, 454 insertions(+), 378 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/pom.xml ---------------------------------------------------------------------- diff --git a/config/pom.xml b/config/pom.xml index 741caac..6719c48 100644 --- a/config/pom.xml +++ b/config/pom.xml @@ -5,7 +5,7 @@ oodt-core org.apache.oodt - 1.1-SNAPSHOT + 1.2-SNAPSHOT ../core/pom.xml 4.0.0 @@ -17,29 +17,25 @@ - org.apache.curator - curator-framework - + args4j + args4j commons-io commons-io - org.springframework - spring-beans - - - org.springframework - spring-context + org.apache.curator + curator-framework - org.springframework - spring-core + org.apache.curator + curator-test - args4j - args4j + junit + junit + test org.slf4j @@ -50,12 +46,16 @@ slf4j-log4j12 - junit - junit + org.springframework + spring-beans - org.apache.curator - curator-test + org.springframework + spring-context + + + org.springframework + spring-core http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/assembly/assembly.xml ---------------------------------------------------------------------- diff --git a/config/src/main/assembly/assembly.xml b/config/src/main/assembly/assembly.xml index 2743cab..bb6f31e 100644 --- a/config/src/main/assembly/assembly.xml +++ b/config/src/main/assembly/assembly.xml @@ -33,8 +33,8 @@ - ${basedir}/src/main/bin - bin + ${basedir}/src/main/scripts + . 755 http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java index da96913..122a78e 100644 --- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java +++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java @@ -37,7 +37,6 @@ import java.util.concurrent.TimeUnit; import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING; import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE; -import static org.apache.oodt.config.Constants.SEPARATOR; /** * Distributed configuration manager implementation. This class make use of a {@link CuratorFramework} instance to http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java new file mode 100644 index 0000000..81fe2af --- /dev/null +++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oodt.config.distributed; + +import org.apache.commons.io.FileUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.oodt.config.Component; +import org.apache.oodt.config.Constants; +import org.apache.oodt.config.distributed.utils.CuratorUtils; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING; +import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE; + +/** + * The class to publish configuration to Zookeeper. When using distributed configuration with OODT, configuration per + * each component type needs to be stored in zookeeper beforehand. This class, provides the means to do that. + * + * @author Imesha Sudasingha + */ +public class DistributedConfigurationPublisher { + + private static final Logger logger = LoggerFactory.getLogger(DistributedConfigurationPublisher.class); + + private Map propertiesFiles; + private Map configFiles; + private String connectString; + private CuratorFramework client; + private ZNodePaths zNodePaths; + private Component component; + + public DistributedConfigurationPublisher(Component component) { + this.component = component; + this.zNodePaths = new ZNodePaths(this.component.getName()); + + if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(ZK_CONNECT_STRING) == null) { + throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " + ZK_CONNECT_STRING + " to be set"); + } + + if (System.getProperty(ZK_PROPERTIES_FILE) != null) { + try { + CuratorUtils.loadZookeeperProperties(); + } catch (IOException e) { + logger.error("Error occurred when loading properties from properties file"); + } + } + + if (System.getProperty(ZK_CONNECT_STRING) == null) { + throw new IllegalArgumentException("Zookeeper requires a proper connect string to connect to zookeeper ensemble"); + } + + connectString = System.getProperty(ZK_CONNECT_STRING); + logger.info("Using zookeeper connect string : {}", connectString); + + startZookeeper(); + } + + /** + * Creates a {@link CuratorFramework} instance and start it. This method will wait a maximum amount of {@link + * Constants.Properties#ZK_STARTUP_TIMEOUT} milli-seconds until the client connects to the zookeeper ensemble. + */ + private void startZookeeper() { + client = CuratorUtils.newCuratorFrameworkClient(connectString, logger); + + client.start(); + logger.info("Curator framework start operation invoked"); + + int startupTimeOutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_STARTUP_TIMEOUT, "30000")); + try { + logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs); + client.blockUntilConnected(startupTimeOutMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException ex) { + logger.error("Interrupted while waiting to connect zookeeper (connectString : {}) : {}", ex, connectString); + } + + if (!client.getZookeeperClient().isConnected()) { + throw new IllegalStateException("Could not connect to ZooKeeper : " + connectString); + } + + logger.info("CuratorFramework client started successfully"); + } + + public void destroy() { + logger.debug("Destroying configuration publisher"); + try { + client.close(); + } catch (Exception e) { + logger.error("Error occurred when trying to close Curator client : {}", e); + } + + logger.info("Configuration publisher destroyed"); + } + + /** + * Publishes the configuration files specified to zookeeper. If an exception is thrown while configuration being + * published, no further publishing attempts will be carried on. Error will be reported to user. + * + * @throws Exception Zookeeper errors + */ + public void publishConfiguration() throws Exception { + logger.debug("Publishing properties files : {}", propertiesFiles); + publishConfiguration(propertiesFiles, true); + logger.info("Properties files published successfully"); + + logger.debug("Publishing config files : {}", configFiles); + publishConfiguration(configFiles, false); + logger.info("Config files published successfully"); + } + + /** + * Verified whether the actual content of the local files specified to be published are 100% similar to the ones + * that has been published and stored in zookeeper at the moment. + * + * @return true | if content are up to date and similar + */ + public boolean verifyPublishedConfiguration() { + try { + return verifyPublishedConfiguration(propertiesFiles, true) && verifyPublishedConfiguration(configFiles, false); + } catch (Exception e) { + logger.error("Error occurred when checking published config", e); + return false; + } + } + + /** + * Removes all the nodes from zookeeper where the configuration corresponding to component {@link #component} is + * stored + * + * @throws Exception zookeeper errors + */ + public void clearConfiguration() throws Exception { + logger.debug("Clearing configuration from zookeeper"); + CuratorUtils.deleteChildNodes(client, zNodePaths.getPropertiesZNodePath()); + CuratorUtils.deleteChildNodes(client, zNodePaths.getConfigurationZNodePath()); + logger.info("Configuration cleared!"); + } + + private void publishConfiguration(Map fileMapping, boolean isProperties) throws Exception { + for (Map.Entry entry : fileMapping.entrySet()) { + String filePath = entry.getKey(); + String relativeZNodePath = entry.getValue(); + logger.info("Publishing configuration {} - {}", filePath, relativeZNodePath); + + String content = getFileContent(filePath); + + String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) : zNodePaths.getConfigurationZNodePath(relativeZNodePath); + if (client.checkExists().forPath(zNodePath) != null) { + byte[] bytes = client.getData().forPath(zNodePath); + String existingData = new String(bytes); + if (existingData.equals(content)) { + logger.warn("{} already exists in zookeeper at {}", filePath, relativeZNodePath); + } else { + Stat stat = client.setData().forPath(zNodePath, content.getBytes()); + if (stat != null) { + logger.info("Published configuration file {} to {}", filePath, relativeZNodePath); + } else { + logger.warn("Unable to publish configuration file {} to {}", filePath, relativeZNodePath); + } + } + } else { + /* + * Creating these ZNodes with parent 'Containers' is important since containers are automatically deleted + * when no child node is present under them. + */ + client.create().creatingParentContainersIfNeeded().forPath(zNodePath, content.getBytes()); + logger.info("Replaced old published configuration at {} with content of file : {}", relativeZNodePath, filePath); + } + } + } + + private boolean verifyPublishedConfiguration(Map fileMapping, boolean isProperties) throws Exception { + boolean noError = true; + for (Map.Entry entry : fileMapping.entrySet()) { + String filePath = entry.getKey(); + String relativeZNodePath = entry.getValue(); + logger.info("Checking published configuration for {} - {}", filePath, relativeZNodePath); + + String originalContent = getFileContent(filePath); + + String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) : zNodePaths.getConfigurationZNodePath(relativeZNodePath); + if (client.checkExists().forPath(zNodePath) == null) { + logger.error("File : {} hasn't been published to ZNode : {}", filePath, relativeZNodePath); + noError = false; + continue; + } + + String publishedContent = new String(client.getData().forPath(zNodePath)); + if (!publishedContent.equals(originalContent)) { + logger.error("Content of local file : {} and content published to {} are not similar", filePath, relativeZNodePath); + noError = false; + continue; + } + + logger.info("{} - {} configuration checked and OK", filePath, relativeZNodePath); + } + + return noError; + } + + private String getFileContent(String file) { + String content; + try { + content = FileUtils.readFileToString(new File(file)); + } catch (IOException e) { + logger.error("Unable to read file : {}", file, e); + throw new IllegalArgumentException("Unable to read content of the file : " + file); + } + + return content; + } + + public Map getPropertiesFiles() { + return propertiesFiles; + } + + public void setPropertiesFiles(Map propertiesFiles) { + this.propertiesFiles = propertiesFiles; + } + + public Map getConfigFiles() { + return configFiles; + } + + public void setConfigFiles(Map configFiles) { + this.configFiles = configFiles; + } + + public ZNodePaths getZNodePaths() { + return zNodePaths; + } + + public Component getComponent() { + return component; + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java new file mode 100644 index 0000000..7a78a8c --- /dev/null +++ b/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oodt.config.distributed.cli; + +import org.apache.oodt.config.Constants; +import org.apache.oodt.config.distributed.DistributedConfigurationPublisher; +import org.kohsuke.args4j.CmdLineException; +import org.kohsuke.args4j.CmdLineParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; + +import java.util.Map; + +import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING; + +public class ConfigPublisher { + + private static final Logger logger = LoggerFactory.getLogger(ConfigPublisher.class); + + public static void main(String[] args) throws Exception { + CmdLineOptions cmdLineOptions = new CmdLineOptions(); + CmdLineParser parser = new CmdLineParser(cmdLineOptions); + + try { + parser.parseArgument(args); + } catch (CmdLineException e) { + System.err.println("There's an error in your command"); + parser.printUsage(System.err); + return; + } + + if (cmdLineOptions.getConnectString() == null && System.getProperty(ZK_CONNECT_STRING) == null) { + System.err.println("Zookeeper connect string is not found"); + parser.printUsage(System.err); + return; + } else { + System.setProperty(ZK_CONNECT_STRING, cmdLineOptions.getConnectString()); + } + + System.out.println("Starting configuration publishing"); + + try { + ApplicationContext applicationContext = new ClassPathXmlApplicationContext(Constants.CONFIG_PUBLISHER_XML); + Map distributedConfigurationPublisher = applicationContext.getBeansOfType(DistributedConfigurationPublisher.class); + + for (Object bean : distributedConfigurationPublisher.values()) { + DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean; + System.out.println(String.format("\nProcessing commands for component : %s", publisher.getComponent())); + + if (cmdLineOptions.isPublish()) { + System.out.println(String.format("Publishing configuration for : %s", publisher.getComponent())); + publisher.publishConfiguration(); + System.out.println(String.format("Published configuration for : %s", publisher.getComponent())); + System.out.println(); + } + + if (cmdLineOptions.isVerify()) { + System.out.println(String.format("Verifying configuration for : %s", publisher.getComponent())); + if (publisher.verifyPublishedConfiguration()) { + System.out.println("OK... Configuration verified"); + System.out.println(String.format("Verified configuration for : %s", publisher.getComponent())); + } else { + System.err.println("ERROR... Published configuration doesn't match the local files. Please check above logs"); + } + System.out.println(); + } + + if (cmdLineOptions.isClear()) { + System.out.println(String.format("Clearing configuration for : %s", publisher.getComponent())); + publisher.clearConfiguration(); + System.out.println(String.format("Cleared configuration for : %s", publisher.getComponent())); + System.out.println(); + } + + publisher.destroy(); + } + } catch (BeansException e) { + logger.error("Error occurred when obtaining configuration publisher beans", e); + throw e; + } catch (Exception e) { + logger.error("Error occurred when publishing configuration to zookeeper", e); + throw e; + } + + logger.info("Exiting CLI ..."); + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java deleted file mode 100644 index 25a45ba..0000000 --- a/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java +++ /dev/null @@ -1,331 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oodt.config.distributed.cli; - -import org.apache.commons.io.FileUtils; -import org.apache.curator.framework.CuratorFramework; -import org.apache.oodt.config.Component; -import org.apache.oodt.config.Constants; -import org.apache.oodt.config.distributed.ZNodePaths; -import org.apache.oodt.config.distributed.utils.CuratorUtils; -import org.apache.zookeeper.data.Stat; -import org.kohsuke.args4j.CmdLineException; -import org.kohsuke.args4j.CmdLineParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.BeansException; -import org.springframework.context.ApplicationContext; -import org.springframework.context.support.ClassPathXmlApplicationContext; - -import java.io.File; -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING; -import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE; - -/** - * The class to publish configuration to Zookeeper. When using distributed configuration with OODT, configuration per - * each component type needs to be stored in zookeeper beforehand. This class, provides the means to do that. - * - * @author Imesha Sudasingha - */ -public class DistributedConfigurationPublisher { - - private static final Logger logger = LoggerFactory.getLogger(DistributedConfigurationPublisher.class); - - private Map propertiesFiles; - private Map configFiles; - private String connectString; - private CuratorFramework client; - private ZNodePaths zNodePaths; - private Component component; - - public DistributedConfigurationPublisher(Component component) { - this.component = component; - this.zNodePaths = new ZNodePaths(this.component.getName()); - - if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(ZK_CONNECT_STRING) == null) { - throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " + ZK_CONNECT_STRING + " to be set"); - } - - if (System.getProperty(ZK_PROPERTIES_FILE) != null) { - try { - CuratorUtils.loadZookeeperProperties(); - } catch (IOException e) { - logger.error("Error occurred when loading properties from properties file"); - } - } - - if (System.getProperty(ZK_CONNECT_STRING) == null) { - throw new IllegalArgumentException("Zookeeper requires a proper connect string to connect to zookeeper ensemble"); - } - - connectString = System.getProperty(ZK_CONNECT_STRING); - logger.info("Using zookeeper connect string : {}", connectString); - - startZookeeper(); - } - - /** - * Creates a {@link CuratorFramework} instance and start it. This method will wait a maximum amount of {@link - * Constants.Properties#ZK_STARTUP_TIMEOUT} milli-seconds until the client connects to the zookeeper ensemble. - */ - private void startZookeeper() { - client = CuratorUtils.newCuratorFrameworkClient(connectString, logger); - - client.start(); - logger.info("Curator framework start operation invoked"); - - int startupTimeOutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_STARTUP_TIMEOUT, "30000")); - try { - logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs); - client.blockUntilConnected(startupTimeOutMs, TimeUnit.MILLISECONDS); - } catch (InterruptedException ex) { - logger.error("Interrupted while waiting to connect zookeeper (connectString : {}) : {}", ex, connectString); - } - - if (!client.getZookeeperClient().isConnected()) { - throw new IllegalStateException("Could not connect to ZooKeeper : " + connectString); - } - - logger.info("CuratorFramework client started successfully"); - } - - public void destroy() { - logger.debug("Destroying configuration publisher"); - try { - client.close(); - } catch (Exception e) { - logger.error("Error occurred when trying to close Curator client : {}", e); - } - - logger.info("Configuration publisher destroyed"); - } - - /** - * Publishes the configuration files specified to zookeeper. If an exception is thrown while configuration being - * published, no further publishing attempts will be carried on. Error will be reported to user. - * - * @throws Exception Zookeeper errors - */ - public void publishConfiguration() throws Exception { - logger.debug("Publishing properties files : {}", propertiesFiles); - publishConfiguration(propertiesFiles, true); - logger.info("Properties files published successfully"); - - logger.debug("Publishing config files : {}", configFiles); - publishConfiguration(configFiles, false); - logger.info("Config files published successfully"); - } - - /** - * Verified whether the actual content of the local files specified to be published are 100% similar to the ones - * that has been published and stored in zookeeper at the moment. - * - * @return true | if content are up to date and similar - */ - public boolean verifyPublishedConfiguration() { - try { - return verifyPublishedConfiguration(propertiesFiles, true) && verifyPublishedConfiguration(configFiles, false); - } catch (Exception e) { - logger.error("Error occurred when checking published config", e); - return false; - } - } - - /** - * Removes all the nodes from zookeeper where the configuration corresponding to component {@link #component} is - * stored - * - * @throws Exception zookeeper errors - */ - public void clearConfiguration() throws Exception { - logger.debug("Clearing configuration from zookeeper"); - CuratorUtils.deleteChildNodes(client, zNodePaths.getPropertiesZNodePath()); - CuratorUtils.deleteChildNodes(client, zNodePaths.getConfigurationZNodePath()); - logger.info("Configuration cleared!"); - } - - private void publishConfiguration(Map fileMapping, boolean isProperties) throws Exception { - for (Map.Entry entry : fileMapping.entrySet()) { - String filePath = entry.getKey(); - String relativeZNodePath = entry.getValue(); - logger.info("Publishing configuration {} - {}", filePath, relativeZNodePath); - - String content = getFileContent(filePath); - - String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) : zNodePaths.getConfigurationZNodePath(relativeZNodePath); - if (client.checkExists().forPath(zNodePath) != null) { - byte[] bytes = client.getData().forPath(zNodePath); - String existingData = new String(bytes); - if (existingData.equals(content)) { - logger.warn("{} already exists in zookeeper at {}", filePath, relativeZNodePath); - } else { - Stat stat = client.setData().forPath(zNodePath, content.getBytes()); - if (stat != null) { - logger.info("Published configuration file {} to {}", filePath, relativeZNodePath); - } else { - logger.warn("Unable to publish configuration file {} to {}", filePath, relativeZNodePath); - } - } - } else { - /* - * Creating these ZNodes with parent 'Containers' is important since containers are automatically deleted - * when no child node is present under them. - */ - client.create().creatingParentContainersIfNeeded().forPath(zNodePath, content.getBytes()); - logger.info("Replaced old published configuration at {} with content of file : {}", relativeZNodePath, filePath); - } - } - } - - private boolean verifyPublishedConfiguration(Map fileMapping, boolean isProperties) throws Exception { - boolean noError = true; - for (Map.Entry entry : fileMapping.entrySet()) { - String filePath = entry.getKey(); - String relativeZNodePath = entry.getValue(); - logger.info("Checking published configuration for {} - {}", filePath, relativeZNodePath); - - String originalContent = getFileContent(filePath); - - String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) : zNodePaths.getConfigurationZNodePath(relativeZNodePath); - if (client.checkExists().forPath(zNodePath) == null) { - logger.error("File : {} hasn't been published to ZNode : {}", filePath, relativeZNodePath); - noError = false; - continue; - } - - String publishedContent = new String(client.getData().forPath(zNodePath)); - if (!publishedContent.equals(originalContent)) { - logger.error("Content of local file : {} and content published to {} are not similar", filePath, relativeZNodePath); - noError = false; - continue; - } - - logger.info("{} - {} configuration checked and OK", filePath, relativeZNodePath); - } - - return noError; - } - - private String getFileContent(String file) { - String content; - try { - content = FileUtils.readFileToString(new File(file)); - } catch (IOException e) { - logger.error("Unable to read file : {}", file, e); - throw new IllegalArgumentException("Unable to read content of the file : " + file); - } - - return content; - } - - public Map getPropertiesFiles() { - return propertiesFiles; - } - - public void setPropertiesFiles(Map propertiesFiles) { - this.propertiesFiles = propertiesFiles; - } - - public Map getConfigFiles() { - return configFiles; - } - - public void setConfigFiles(Map configFiles) { - this.configFiles = configFiles; - } - - public ZNodePaths getZNodePaths() { - return zNodePaths; - } - - public static void main(String[] args) throws Exception { - CmdLineOptions cmdLineOptions = new CmdLineOptions(); - CmdLineParser parser = new CmdLineParser(cmdLineOptions); - - try { - parser.parseArgument(args); - } catch (CmdLineException e) { - System.err.println("There's an error in your command"); - parser.printUsage(System.err); - return; - } - - if (cmdLineOptions.getConnectString() == null && System.getProperty(ZK_CONNECT_STRING) == null) { - System.err.println("Zookeeper connect string is not found"); - parser.printUsage(System.err); - return; - } else { - System.setProperty(ZK_CONNECT_STRING, cmdLineOptions.getConnectString()); - } - - System.out.println("Starting configuration publishing"); - - try { - ApplicationContext applicationContext = new ClassPathXmlApplicationContext(Constants.CONFIG_PUBLISHER_XML); - Map distributedConfigurationPublisher = applicationContext.getBeansOfType(DistributedConfigurationPublisher.class); - - for (Object bean : distributedConfigurationPublisher.values()) { - DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean; - System.out.println(String.format("\nProcessing commands for component : %s", publisher.getComponent())); - - if (cmdLineOptions.isPublish()) { - System.out.println(String.format("Publishing configuration for : %s", publisher.getComponent())); - publisher.publishConfiguration(); - System.out.println(String.format("Published configuration for : %s", publisher.getComponent())); - System.out.println(); - } - - if (cmdLineOptions.isVerify()) { - System.out.println(String.format("Verifying configuration for : %s", publisher.getComponent())); - if (publisher.verifyPublishedConfiguration()) { - System.out.println("OK... Configuration verified"); - System.out.println(String.format("Verified configuration for : %s", publisher.getComponent())); - } else { - System.err.println("ERROR... Published configuration doesn't match the local files. Please check above logs"); - } - System.out.println(); - } - - if (cmdLineOptions.isClear()) { - System.out.println(String.format("Clearing configuration for : %s", publisher.getComponent())); - publisher.clearConfiguration(); - System.out.println(String.format("Cleared configuration for : %s", publisher.getComponent())); - System.out.println(); - } - - publisher.destroy(); - } - } catch (BeansException e) { - logger.error("Error occurred when obtaining configuration publisher beans", e); - throw e; - } catch (Exception e) { - logger.error("Error occurred when publishing configuration to zookeeper", e); - throw e; - } - - logger.info("Exiting CLI ..."); - } - - public Component getComponent() { - return component; - } -} http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/java/org/apache/oodt/config/distributed/utils/FilePathUtils.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/distributed/utils/FilePathUtils.java b/config/src/main/java/org/apache/oodt/config/distributed/utils/FilePathUtils.java index 5b242c2..cfb5ff9 100644 --- a/config/src/main/java/org/apache/oodt/config/distributed/utils/FilePathUtils.java +++ b/config/src/main/java/org/apache/oodt/config/distributed/utils/FilePathUtils.java @@ -32,7 +32,11 @@ public class FilePathUtils { } public static String fixForComponentHome(Component component, String suffixPath) { - String prefix = System.getenv().get(component.getHome()); + String prefix = System.getProperty(component.getHome()); + if (prefix == null) { + prefix = System.getenv().get(component.getHome()); + } + StringBuilder path = new StringBuilder(); if (prefix != null && !prefix.trim().isEmpty()) { prefix = prefix.trim(); http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/resources/etc/config-publisher.xml ---------------------------------------------------------------------- diff --git a/config/src/main/resources/etc/config-publisher.xml b/config/src/main/resources/etc/config-publisher.xml index 443a7a5..88ba33b 100644 --- a/config/src/main/resources/etc/config-publisher.xml +++ b/config/src/main/resources/etc/config-publisher.xml @@ -20,21 +20,21 @@ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd"> - + - + - - - - - - + + + + + + http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/resources/etc/log4j.xml ---------------------------------------------------------------------- diff --git a/config/src/main/resources/etc/log4j.xml b/config/src/main/resources/etc/log4j.xml index 17d6e8e..8cc1919 100644 --- a/config/src/main/resources/etc/log4j.xml +++ b/config/src/main/resources/etc/log4j.xml @@ -37,7 +37,7 @@ - + @@ -45,6 +45,9 @@ + + + http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/scripts/conf-publisher ---------------------------------------------------------------------- diff --git a/config/src/main/scripts/conf-publisher b/config/src/main/scripts/conf-publisher new file mode 100644 index 0000000..618af4b --- /dev/null +++ b/config/src/main/scripts/conf-publisher @@ -0,0 +1,36 @@ +#!/bin/sh +#/* +# * Licensed to the Apache Software Foundation (ASF) under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The ASF licenses this file to You under the Apache License, Version 2.0 +# * (the "License"); you may not use this file except in compliance with +# * the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +[ -f /etc/sysconfig/java ] && . /etc/sysconfig/java + +CONF_HOME=. + +for file in `find ${CONF_HOME}/lib/*.jar`; do + LIB_DEPS="${file}:${LIB_DEPS}" +done + +LIB_DEPS="${CONF_HOME}/etc/log4j.xml:${LIB_DEPS}" + +echo -n "Starting Configuration Publisher" +$JAVA_HOME/bin/java \ + -cp ${LIB_DEPS} \ + -Dlog4j.configuration=etc/log4j.xml \ + org.apache.oodt.config.distributed.cli.ConfigPublisher "$@" +echo "OK" +sleep 1 +exit 0 http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java ---------------------------------------------------------------------- diff --git a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java index 08ecdc1..9052ac3 100644 --- a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java +++ b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java @@ -19,14 +19,12 @@ package org.apache.oodt.config.distributed; import org.apache.commons.io.FileUtils; import org.apache.oodt.config.ConfigurationManager; -import org.apache.oodt.config.distributed.cli.DistributedConfigurationPublisher; +import org.apache.oodt.config.distributed.cli.ConfigPublisher; import org.apache.oodt.config.distributed.utils.FilePathUtils; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.fail; -import static org.junit.Assert.assertNotNull; import org.springframework.context.ApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; @@ -42,6 +40,7 @@ import java.util.Set; import static org.apache.oodt.config.Constants.CONFIG_PUBLISHER_XML; import static org.apache.oodt.config.Constants.SEPARATOR; +import static org.junit.Assert.fail; /** * Testing the {@link DistributedConfigurationManager} whether it is downloading and storing the configuration correctly @@ -57,7 +56,7 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf public static void setUp() throws Exception { AbstractDistributedConfigurationTest.setUp(); - DistributedConfigurationPublisher.main(new String[]{ + ConfigPublisher.main(new String[]{ "-connectString", zookeeper.getConnectString(), "-publish" }); @@ -68,6 +67,8 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf publishers = new ArrayList<>(distributedConfigurationPublishers.values().size()); for (Object bean : distributedConfigurationPublishers.values()) { DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean; + + System.setProperty(publisher.getComponent().getHome(), "."); publishers.add(publisher); } } @@ -84,11 +85,10 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf Properties properties = new Properties(); try (InputStream in = new FileInputStream(originalFile)) { properties.load(in); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); } - catch (Exception e){ - e.printStackTrace(); - fail(e.getMessage()); - } for (String key : properties.stringPropertyNames()) { Assert.assertEquals(properties.getProperty(key), System.getProperty(key)); @@ -98,7 +98,7 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf fileName = fileName.startsWith(SEPARATOR) ? fileName.substring(SEPARATOR.length()) : fileName; fileName = FilePathUtils.fixForComponentHome(publisher.getComponent(), fileName); File downloadedFile = new File(fileName); - Assert.assertNotNull(downloadedFile); + Assert.assertNotNull(downloadedFile); Assert.assertTrue(downloadedFile.exists()); } @@ -126,7 +126,10 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf String fileName = entry.getValue(); fileName = fileName.startsWith(SEPARATOR) ? fileName.substring(1) : fileName; - String prefixPath = System.getenv(publisher.getComponent().getHome()); + String prefixPath = System.getProperty(publisher.getComponent().getHome()); + if (prefixPath == null) { + prefixPath = System.getenv(publisher.getComponent().getHome()); + } String confDir = prefixPath != null && !prefixPath.trim().isEmpty() ? prefixPath.trim() + SEPARATOR + fileName.split(SEPARATOR)[0] : fileName.split(SEPARATOR)[0]; @@ -135,7 +138,7 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf } } - DistributedConfigurationPublisher.main(new String[]{ + ConfigPublisher.main(new String[]{ "-connectString", zookeeper.getConnectString(), "-clear" }); http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java ---------------------------------------------------------------------- diff --git a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java index 4f1552b..2c130db 100644 --- a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java +++ b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java @@ -18,7 +18,7 @@ package org.apache.oodt.config.distributed; import org.apache.commons.io.FileUtils; -import org.apache.oodt.config.distributed.cli.DistributedConfigurationPublisher; +import org.apache.oodt.config.distributed.cli.ConfigPublisher; import org.junit.Assert; import org.junit.Test; import org.springframework.context.ApplicationContext; @@ -41,7 +41,7 @@ public class DistributedConfigurationPublisherTest extends AbstractDistributedCo @Test public void publishConfigurationTest() throws Exception { // Publishing configuration through CLI and verifying whether they were stored correctly - DistributedConfigurationPublisher.main(new String[]{ + ConfigPublisher.main(new String[]{ "-connectString", zookeeper.getConnectString(), "-publish", "-verify" @@ -82,7 +82,7 @@ public class DistributedConfigurationPublisherTest extends AbstractDistributedCo } // Clearing configuration through CLI and checking whether the configuration has actually been gone - DistributedConfigurationPublisher.main(new String[]{ + ConfigPublisher.main(new String[]{ "-connectString", zookeeper.getConnectString(), "-clear" }); http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/test/resources/etc/config-publisher.xml ---------------------------------------------------------------------- diff --git a/config/src/test/resources/etc/config-publisher.xml b/config/src/test/resources/etc/config-publisher.xml index 89e16f3..ecae033 100644 --- a/config/src/test/resources/etc/config-publisher.xml +++ b/config/src/test/resources/etc/config-publisher.xml @@ -19,7 +19,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd"> - + @@ -38,7 +38,7 @@ - +