oodt-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mattm...@apache.org
Subject [48/52] [abbrv] oodt git commit: Improvements to filemgr and config-publisher bash scripts
Date Wed, 30 Aug 2017 16:40:17 GMT
Improvements to filemgr and config-publisher bash scripts


Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/9ee8af48
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/9ee8af48
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/9ee8af48

Branch: refs/heads/master
Commit: 9ee8af48ba45904a44d4e32b3030e8ad7d0bee21
Parents: 57a9235
Author: Imesha Sudasingha <imesha.sudasingha@gmail.com>
Authored: Thu Aug 10 18:32:27 2017 +0530
Committer: Imesha Sudasingha <imesha.sudasingha@gmail.com>
Committed: Thu Aug 10 18:32:27 2017 +0530

----------------------------------------------------------------------
 .../java/org/apache/oodt/config/Component.java  |  4 +--
 .../config/ConfigurationManagerFactory.java     | 37 ++++++++++++++++----
 .../java/org/apache/oodt/config/Constants.java  |  3 ++
 config/src/main/resources/log4j.xml             |  1 -
 core/pom.xml                                    | 11 +++---
 filemgr/pom.xml                                 |  6 +++-
 filemgr/src/main/bin/filemgr                    |  6 ++--
 .../TestDistributedXmlRpcFileManager.java       |  1 +
 .../distributed/config/filemgr.properties       |  8 ++---
 9 files changed, 54 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/config/src/main/java/org/apache/oodt/config/Component.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/Component.java b/config/src/main/java/org/apache/oodt/config/Component.java
index e2ea223..2c7123e 100644
--- a/config/src/main/java/org/apache/oodt/config/Component.java
+++ b/config/src/main/java/org/apache/oodt/config/Component.java
@@ -24,8 +24,8 @@ package org.apache.oodt.config;
  * @author Imesha Sudasingha
  */
 public enum Component {
-    FILE_MANAGER("filemgr", "OODT_FILEMGR_HOME"),
-    RESOURCE_MANAGER("resmgr", "OODT_RESMGR_HOME");
+    FILE_MANAGER("filemgr", "FILEMGR_HOME"),
+    RESOURCE_MANAGER("resmgr", "RESMGR_HOME");
 
     /** Shorthand name of the component. Will be used when creating ZNodes in zookeeper */
     String name;

http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java
b/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java
index 9eecfaa..f3a5dc5 100644
--- a/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java
+++ b/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java
@@ -17,6 +17,7 @@
 
 package org.apache.oodt.config;
 
+import org.apache.oodt.config.Constants.Env;
 import org.apache.oodt.config.distributed.DistributedConfigurationManager;
 import org.apache.oodt.config.standalone.StandaloneConfigurationManager;
 import org.slf4j.Logger;
@@ -24,7 +25,9 @@ import org.slf4j.LoggerFactory;
 
 import java.util.List;
 
+import static org.apache.oodt.config.Constants.Env.CONNECT_STRING;
 import static org.apache.oodt.config.Constants.Properties.ENABLE_DISTRIBUTED_CONFIGURATION;
+import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
 
 /**
  * Factory class to be used to get the {@link ConfigurationManager} instances accordingly.
@@ -40,18 +43,38 @@ public class ConfigurationManagerFactory {
     }
 
     /**
-     * Returns the {@link ConfigurationManager} to be used by the calling class. Whether
to use the standalone version or
-     * the distributed version of the configuration manager will be determined by the value
of the property
+     * Returns the {@link ConfigurationManager} to be used by the calling class. Whether
to use the standalone version
+     * or the distributed version of the configuration manager will be determined by the
value of the property
      * <pre>org.apache.oodt.config.zookeeper == true</pre>
      *
-     * @param component       Name of the OODT component, to which the created configuration
manager instance will be providing
-     *                        configuration support
-     * @param propertiesFiles List of <pre>.properties</pre> files which are
to be used in the case of standalone configuration
-     *                        management
+     * @param component       Name of the OODT component, to which the created configuration
manager instance will be
+     *                        providing configuration support
+     * @param propertiesFiles List of <pre>.properties</pre> files which are
to be used in the case of standalone
+     *                        configuration management
      * @return ConfigurationManager instance to used by the corresponding component.
      */
     public static ConfigurationManager getConfigurationManager(Component component, List<String>
propertiesFiles) {
-        if (Boolean.getBoolean(ENABLE_DISTRIBUTED_CONFIGURATION)) {
+        boolean isDistributed = Boolean.getBoolean(ENABLE_DISTRIBUTED_CONFIGURATION);
+        if (!isDistributed) {
+            String env = System.getenv(Env.ENABLE_DISTRIBUTED_CONFIGURATION);
+            if (env != null) {
+                isDistributed = Boolean.parseBoolean(env);
+            }
+        }
+
+        if (isDistributed) {
+            String connectString = System.getProperty(ZK_CONNECT_STRING);
+            if (connectString == null) {
+                connectString = System.getenv(CONNECT_STRING);
+            }
+
+            if (connectString == null) {
+                throw new IllegalArgumentException(
+                        String.format("%s environment variable need to be set for distributed
configuration management", CONNECT_STRING));
+            }
+
+            System.setProperty(ZK_CONNECT_STRING, connectString);
+
             logger.info("Using distributed configuration management for {}", component);
             return new DistributedConfigurationManager(component);
         }

http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/config/src/main/java/org/apache/oodt/config/Constants.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/Constants.java b/config/src/main/java/org/apache/oodt/config/Constants.java
index 611f81e..9cd217f 100644
--- a/config/src/main/java/org/apache/oodt/config/Constants.java
+++ b/config/src/main/java/org/apache/oodt/config/Constants.java
@@ -43,6 +43,9 @@ public class Constants {
     public static class Env {
         /** Environment variable name to specify OODT project name */
         public static final String OODT_PROJECT = "OODT_PROJECT";
+        /** The environment variable to be set, if to enable distributed configuration management
*/
+        public static final String ENABLE_DISTRIBUTED_CONFIGURATION = "OODT_DISTRIBUTED_CONF";
+        public static final String CONNECT_STRING = "ZK_CONNECT_STRING";
     }
 
     public static class Properties {

http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/config/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/config/src/main/resources/log4j.xml b/config/src/main/resources/log4j.xml
index 8cc1919..0cd8348 100644
--- a/config/src/main/resources/log4j.xml
+++ b/config/src/main/resources/log4j.xml
@@ -39,7 +39,6 @@
     <root>
         <priority value="INFO"/>
         <appender-ref ref="console"/>
-        <appender-ref ref="file"/>
     </root>
 
     <logger name="org.apache.zookeeper">

http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index b9f07e1..5e99f9e 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -253,6 +253,11 @@ the License.
         <version>3.3.0</version>
       </dependency>
       <dependency>
+        <groupId>org.apache.zookeeper</groupId>
+        <artifactId>zookeeper</artifactId>
+        <version>3.5.1-alpha</version>
+      </dependency>
+      <dependency>
         <groupId>org.apache.cxf</groupId>
         <artifactId>cxf-rt-frontend-jaxrs</artifactId>
         <version>2.6.0</version>
@@ -412,12 +417,6 @@ the License.
         <groupId>org.apache.solr</groupId>
         <artifactId>solr-solrj</artifactId>
         <version>6.2.1</version>
-        <exclusions>
-          <exclusion>
-            <groupId>org.apache.zookeeper</groupId>
-            <artifactId>zookeeper</artifactId>
-          </exclusion>
-        </exclusions>
       </dependency>
       <dependency>
         <groupId>org.apache.tika</groupId>

http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/filemgr/pom.xml
----------------------------------------------------------------------
diff --git a/filemgr/pom.xml b/filemgr/pom.xml
index 4fadcb8..2086dc1 100644
--- a/filemgr/pom.xml
+++ b/filemgr/pom.xml
@@ -163,6 +163,10 @@
           <groupId>org.apache.solr</groupId>
           <artifactId>solr-lucene-core</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.zookeeper</groupId>
+          <artifactId>zookeeper</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -236,7 +240,7 @@
             </property>
           </systemProperties>
           <environmentVariables>
-            <OODT_FILEMGR_HOME>${project.basedir}</OODT_FILEMGR_HOME>
+            <FILEMGR_HOME>${project.basedir}</FILEMGR_HOME>
             <OODT_PROJECT>primary</OODT_PROJECT>
           </environmentVariables>
           <forkedProcessTimeoutInSeconds>0</forkedProcessTimeoutInSeconds>

http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/filemgr/src/main/bin/filemgr
----------------------------------------------------------------------
diff --git a/filemgr/src/main/bin/filemgr b/filemgr/src/main/bin/filemgr
index 1a4af47..02f5b90 100644
--- a/filemgr/src/main/bin/filemgr
+++ b/filemgr/src/main/bin/filemgr
@@ -34,7 +34,7 @@ else
 fi
 
 export JAVA_HOME
-FILEMGR_HOME=..
+FILEMGR_HOME=`cd ..; pwd`
 export FILEMGR_HOME
 RUN_HOME=${FILEMGR_HOME}/../run
 export RUN_HOME
@@ -52,6 +52,8 @@ for file in `find ../lib/*.jar`; do
      LIB_DEPS="${file}:${LIB_DEPS}"
 done
 
+LIB_DEPS="${FILEMGR_HOME}/etc/log4j.xml:${LIB_DEPS}"
+
 if [ ! -z $OODT_DISTRIBUTED_CONF ]
     then
     if [ ! -z $ZK_CONNECT_STRING ]
@@ -73,7 +75,7 @@ case "$1" in
         $JAVA_HOME/bin/java \
         	-cp ${LIB_DEPS} \
         	${DISTRIBUTED_CONF_PROPERTIES} \
-        	-Dlog4j.configuration=./${FILEMGR_HOME}/etc/log4j.xml \
+        	-Dlog4j.configuration=log4j.xml \
         	-Djava.util.logging.config.file=${FILEMGR_HOME}/etc/logging.properties \
     	    -Dorg.apache.oodt.cas.filemgr.properties=${CAS_FILEMGR_PROPS} \
         	org.apache.oodt.cas.filemgr.system.XmlRpcFileManager \

http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java
----------------------------------------------------------------------
diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java
b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java
index afbac03..3dd158b 100644
--- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java
+++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java
@@ -44,6 +44,7 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+// TODO: 8/10/17 Cleanup Dirs once done
 public class TestDistributedXmlRpcFileManager extends AbstractDistributedConfigurationTest
{
 
     private static final int FM_PORT = 9001;

http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/filemgr/src/test/resources/distributed/config/filemgr.properties
----------------------------------------------------------------------
diff --git a/filemgr/src/test/resources/distributed/config/filemgr.properties b/filemgr/src/test/resources/distributed/config/filemgr.properties
index 50d09a7..2ca5096 100644
--- a/filemgr/src/test/resources/distributed/config/filemgr.properties
+++ b/filemgr/src/test/resources/distributed/config/filemgr.properties
@@ -63,7 +63,7 @@ org.apache.oodt.cas.filemgr.catalog.science.jdbc.pass=
 org.apache.oodt.cas.filemgr.catalog.science.jdbc.driver=org.hsqldb.jdbc.JDBCDriver
 
 # lucene catalog configuration
-org.apache.oodt.cas.filemgr.catalog.lucene.idxPath=file:target/tmp
+org.apache.oodt.cas.filemgr.catalog.lucene.idxPath=target/tmp
 org.apache.oodt.cas.filemgr.catalog.lucene.pageSize=20
 org.apache.oodt.cas.filemgr.catalog.lucene.commitLockTimeout.seconds=60
 org.apache.oodt.cas.filemgr.catalog.lucene.writeLockTimeout.seconds=60
@@ -90,10 +90,10 @@ org.apache.oodt.cas.filemgr.repositorymgr.science.jdbc.pass=
 org.apache.oodt.cas.filemgr.repositorymgr.science.jdbc.driver=org.hsqldb.jdbc.JDBCDriver
 
 # XML repository manager configuration
-org.apache.oodt.cas.filemgr.repositorymgr.dirs=file:[OODT_FILEMGR_HOME]/target/filemgr/policy/oodt
+org.apache.oodt.cas.filemgr.repositorymgr.dirs=file:[FILEMGR_HOME]/target/filemgr/policy/oodt
 
 # XML validation layer configuration
-org.apache.oodt.cas.filemgr.validation.dirs=file:[OODT_FILEMGR_HOME]/target/filemgr/policy/oodt
+org.apache.oodt.cas.filemgr.validation.dirs=file:[FILEMGR_HOME]/target/filemgr/policy/oodt
 
 # set the following property to 'true' to allow dynamic metadata fields,
 # effectively bypassing the validation layer.
@@ -125,7 +125,7 @@ org.apache.oodt.cas.filemgr.datatransfer.s3.access.key=s3_access_key
 org.apache.oodt.cas.filemgr.datatransfer.s3.secret.key=s3_secret_key
 
 # location of Mime-Type repository
-org.apache.oodt.cas.filemgr.mime.type.repository=[OODT_FILEMGR_HOME]/target/filemgr/policy/mime-types.xml
+org.apache.oodt.cas.filemgr.mime.type.repository=[FILEMGR_HOME]/target/filemgr/policy/mime-types.xml
 
 # tells the file manager system layer to include product instance metadata
 # NOTE: here are the expected field mappings


Mime
View raw message