falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkat...@apache.org
Subject git commit: FALCON-65 Avoid bundling compilation path dependent configs in artifact and allow for config scan to seamlessly fallback to classpath if not found in default conf location. Contributed by Srikanth Sundarrajan
Date Mon, 29 Jul 2013 17:24:04 GMT
Updated Branches:
  refs/heads/master 027918ad6 -> 46dcea2f2


FALCON-65 Avoid bundling compilation path dependent configs in artifact and allow for config
scan to seamlessly fallback to classpath if not found in default conf location. Contributed
by Srikanth Sundarrajan


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/46dcea2f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/46dcea2f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/46dcea2f

Branch: refs/heads/master
Commit: 46dcea2f22bad1fac9491e276458dd4128dd0647
Parents: 027918a
Author: Venkatesh Seetharam <venkatesh@apache.org>
Authored: Mon Jul 29 10:25:55 2013 -0700
Committer: Venkatesh Seetharam <venkatesh@apache.org>
Committed: Mon Jul 29 10:25:55 2013 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |   6 +-
 .../falcon/util/ApplicationProperties.java      | 147 +++++++++++--------
 common/src/main/resources/log4j.xml             |   2 +-
 common/src/main/resources/startup.properties    |   2 +-
 .../falcon/util/ApplicationPropertiesTest.java  | 100 +++++++++++++
 .../falcon/util/StartupPropertiesTest.java      |  42 ------
 common/src/test/resources/classpath.properties  |  21 +++
 pom.xml                                         |  22 +++
 prism/src/main/resources/log4j.xml              |   2 +-
 src/bin/falcon-start                            |   2 +-
 src/conf/log4j.xml                              |   2 +-
 src/conf/startup.properties                     |   2 +-
 webapp/pom.xml                                  |  15 ++
 webapp/src/main/resources/log4j.xml             |   2 +-
 14 files changed, 254 insertions(+), 113 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/46dcea2f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 51b4174..b19c015 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -11,11 +11,15 @@ Trunk (Unreleased)
   OPTIMIZATIONS
 
   BUG FIXES
-    FALCON-65 The identifier regex for oozie bundle,coordinator, workflow 
+    FALCON-61 The identifier regex for oozie bundle,coordinator, workflow
     pattern is incorrect. (Suhas V via Srikanth Sundarrajan)
 
 Release Version: 0.3
 
+    FALCON-65 Avoid bundling compilation path dependent configs in artifact
+    and allow for config scan to seamlessly fallback to classpath if not found
+    in default conf location. (Srikanth Sundarrajan via Venkatesh Seetharam)
+
     FALCON-58 Gaps in release source / binary tarball. (Srikanth Sundarrajan)
 
     FALCON-57 Build fails if the source is not in scm. (Srikanth Sundarrajan)

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/46dcea2f/common/src/main/java/org/apache/falcon/util/ApplicationProperties.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/util/ApplicationProperties.java b/common/src/main/java/org/apache/falcon/util/ApplicationProperties.java
index 3746729..5551bec 100644
--- a/common/src/main/java/org/apache/falcon/util/ApplicationProperties.java
+++ b/common/src/main/java/org/apache/falcon/util/ApplicationProperties.java
@@ -18,6 +18,7 @@
 
 package org.apache.falcon.util;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.falcon.FalconException;
 import org.apache.falcon.expression.ExpressionHelper;
 import org.apache.log4j.Logger;
@@ -27,6 +28,7 @@ import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URL;
 import java.util.HashSet;
 import java.util.Properties;
 import java.util.Set;
@@ -38,92 +40,111 @@ public abstract class ApplicationProperties extends Properties {
 
     private static final Logger LOG = Logger.getLogger(ApplicationProperties.class);
 
-    protected enum LocationType {
-        FILE, HOME, CLASSPATH
-    }
-
     protected abstract String getPropertyFile();
 
-    private String propertyFile;
-    private LocationType location;
     private String domain;
 
     protected ApplicationProperties() throws FalconException {
-        initialize();
+        init();
+    }
+
+    protected void init() throws FalconException {
+        setDomain(System.getProperty("falcon.domain", System.getenv("FALCON_DOMAIN")));
         loadProperties();
     }
 
+    protected void setDomain(String domain) {
+        this.domain = domain;
+    }
+
     public String getDomain() {
         return domain;
     }
 
-    protected void initialize() {
-        String propFile = getPropertyFile();
-        String userHome = System.getProperty("user.home");
+    protected void loadProperties() throws FalconException {
+        String propertyFileName = getPropertyFile();
         String confDir = System.getProperty("config.location");
-        domain = System.getProperty("falcon.domain", System.getenv("FALCON_DOMAIN"));
-
-        if (confDir == null && new File(userHome, propFile).exists()) {
-            LOG.info("config.location is not set, and property file found in home dir" +
userHome + "/" + propFile);
-            location = LocationType.HOME;
-            propertyFile = new File(userHome, propFile).getAbsolutePath();
-        } else if (confDir != null) {
-            LOG.info("config.location is set, using " + confDir + "/" + propFile);
-            location = LocationType.FILE;
-            propertyFile = new File(confDir, propFile).getAbsolutePath();
-        } else {
-            LOG.info("config.location is not set, properties file not present in "
-                    + "user home dir, falling back to classpath for "
-                    + propFile);
-            location = LocationType.CLASSPATH;
-            propertyFile = propFile;
-        }
+        loadProperties(propertyFileName, confDir);
     }
 
-    void loadProperties() throws FalconException {
-        InputStream resource;
+    /**
+     * This method reads the given properties file in the following order:
+     * config.location & classpath. It falls back in that specific order.
+     *
+     * @throws FalconException
+     */
+    protected void loadProperties(String propertyFileName, String confDir) throws FalconException
{
         try {
-            if (location == LocationType.CLASSPATH) {
-                Class clazz = ApplicationProperties.class;
-                if (clazz.getResource(propertyFile) != null) {
-                    LOG.info("Property file being loaded from " + clazz.getResource(propertyFile));
-                    resource = clazz.getResourceAsStream(propertyFile);
-                } else {
-                    LOG.info("Property file being loaded from " + clazz.getResource("/" +
propertyFile));
-                    resource = clazz.getResourceAsStream("/" + propertyFile);
-                }
-            } else {
-                resource = new FileInputStream(propertyFile);
+            InputStream resourceAsStream = checkConfigLocation(propertyFileName, confDir);
+
+            //Fallback to classpath
+            if (resourceAsStream == null) {
+                resourceAsStream = checkClassPath(propertyFileName);
             }
 
-            if (resource == null) {
-                throw new FileNotFoundException(propertyFile + " not found in " + location);
-            } else {
+            if (resourceAsStream != null) {
                 try {
-                    LOG.info("Loading properties from " + propertyFile);
-                    Properties origProps = new Properties();
-                    origProps.load(resource);
-                    if (domain == null) {
-                        domain = origProps.getProperty("*.domain");
-                        if (domain == null) {
-                            throw new FalconException("Domain is not set!");
-                        }
-                    }
-                    LOG.info("Initializing properties with domain " + domain);
-
-                    Set<String> keys = getKeys(origProps.keySet());
-                    for (String key : keys) {
-                        String value = origProps.getProperty(domain + "." + key, origProps.getProperty("*."
+ key));
-                        value = ExpressionHelper.substitute(value);
-                        LOG.debug(key + "=" + value);
-                        put(key, value);
-                    }
+                    doLoadProperties(resourceAsStream);
+                    return;
                 } finally {
-                    resource.close();
+                    IOUtils.closeQuietly(resourceAsStream);
                 }
             }
+            throw new FileNotFoundException("Unable to find: " + propertyFileName);
         } catch (IOException e) {
-            throw new FalconException(e);
+            throw new FalconException("Error loading properties file: " + getPropertyFile(),
e);
+        }
+    }
+
+    private InputStream checkConfigLocation(String propertyFileName, String confDir)
+        throws FileNotFoundException {
+
+        InputStream resourceAsStream = null;
+        if (confDir != null) {
+            File fileToLoad = new File(confDir, propertyFileName);
+            if (fileToLoad.exists()) {
+                LOG.info("config.location is set, using: " + confDir + "/" + propertyFileName);
+                resourceAsStream = new FileInputStream(fileToLoad);
+            }
+        }
+        return resourceAsStream;
+    }
+
+    private InputStream checkClassPath(String propertyFileName) {
+
+        InputStream resourceAsStream = null;
+        Class clazz = ApplicationProperties.class;
+        URL resource = clazz.getResource("/" + propertyFileName);
+        if (resource != null) {
+            LOG.info("Fallback to classpath for: " + resource);
+            resourceAsStream = clazz.getResourceAsStream("/" + propertyFileName);
+        } else {
+            resource = clazz.getResource(propertyFileName);
+            if (resource != null) {
+                LOG.info("Fallback to classpath for: " + resource);
+                resourceAsStream = clazz.getResourceAsStream(propertyFileName);
+            }
+        }
+        return resourceAsStream;
+    }
+
+    private void doLoadProperties(InputStream resourceAsStream) throws IOException, FalconException
{
+        Properties origProps = new Properties();
+        origProps.load(resourceAsStream);
+        if (domain == null) {
+            domain = origProps.getProperty("*.domain");
+            if (domain == null) {
+                throw new FalconException("Domain is not set!");
+            }
+        }
+
+        LOG.info("Initializing properties with domain " + domain);
+        Set<String> keys = getKeys(origProps.keySet());
+        for (String key : keys) {
+            String value = origProps.getProperty(domain + "." + key, origProps.getProperty("*."
+ key));
+            value = ExpressionHelper.substitute(value);
+            LOG.debug(key + "=" + value);
+            put(key, value);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/46dcea2f/common/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/common/src/main/resources/log4j.xml b/common/src/main/resources/log4j.xml
index 4183181..959e26c 100644
--- a/common/src/main/resources/log4j.xml
+++ b/common/src/main/resources/log4j.xml
@@ -63,7 +63,7 @@
         </layout>
     </appender>
 
-    <logger name="org.apache.falcon">
+    <logger name="org.apache.falcon" additivity="false">
         <level value="debug"/>
         <appender-ref ref="FILE"/>
     </logger>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/46dcea2f/common/src/main/resources/startup.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties
index 4ff00d7..21c4f3f 100644
--- a/common/src/main/resources/startup.properties
+++ b/common/src/main/resources/startup.properties
@@ -46,7 +46,7 @@
 
 debug.config.store.uri=file://${user.dir}/target/store
 debug.config.oozie.conf.uri=${user.dir}/target/oozie
-debug.system.lib.location=${user.dir}/webapp/target/falcon-webapp-${pom.version}/WEB-INF/lib
+debug.system.lib.location=${user.dir}/webapp/target/dependency
 debug.broker.url=vm://localhost
 debug.retry.recorder.path=${user.dir}/target/retry
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/46dcea2f/common/src/test/java/org/apache/falcon/util/ApplicationPropertiesTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/util/ApplicationPropertiesTest.java b/common/src/test/java/org/apache/falcon/util/ApplicationPropertiesTest.java
new file mode 100644
index 0000000..c11481a
--- /dev/null
+++ b/common/src/test/java/org/apache/falcon/util/ApplicationPropertiesTest.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.util;
+
+import org.apache.falcon.FalconException;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+
+/**
+ * Test for Application properties test.
+ */
+public class ApplicationPropertiesTest {
+
+    @Test
+    public void testConfigLocation() throws Exception {
+        File target = new File("target");
+        if (!target.exists()) {
+            target = new File("common/target");
+        }
+
+        FileOutputStream out = new FileOutputStream(new File(target, "config.properties"));
+        out.write("*.domain=unittest\n".getBytes());
+        out.write("unittest.test=hello world\n".getBytes());
+        out.close();
+        ApplicationProperties configLocation = new ConfigLocation();
+        configLocation.loadProperties("config.properties", target.getAbsolutePath());
+        Assert.assertEquals(configLocation.getDomain(), "unittest");
+        Assert.assertEquals(configLocation.get("test"), "hello world");
+    }
+
+    @Test
+    public void testClassPathLocation() throws Exception {
+        ApplicationProperties classPathLocation = new ClassPathLocation();
+        classPathLocation.loadProperties("classpath.properties", null);
+        Assert.assertEquals(classPathLocation.getDomain(), "unittest");
+        Assert.assertEquals(classPathLocation.get("test"), "hello world");
+    }
+
+    @Test (expectedExceptions = FalconException.class)
+    public void testMissingLocation() throws FalconException {
+        new MissingLocation().loadProperties();
+    }
+
+    private class ConfigLocation extends ApplicationProperties {
+
+        protected ConfigLocation() throws FalconException {
+        }
+
+        protected void init() {}
+
+        @Override
+        protected String getPropertyFile() {
+            return "config.properties";
+        }
+    }
+
+    private class ClassPathLocation extends ApplicationProperties {
+
+        protected ClassPathLocation() throws FalconException {
+        }
+
+        protected void init() {}
+
+        @Override
+        protected String getPropertyFile() {
+            return "classpath.properties";
+        }
+    }
+
+    private class MissingLocation extends ApplicationProperties {
+
+        protected MissingLocation() throws FalconException {
+        }
+
+        @Override
+        protected String getPropertyFile() {
+            return "missing.properties";
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/46dcea2f/common/src/test/java/org/apache/falcon/util/StartupPropertiesTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/util/StartupPropertiesTest.java b/common/src/test/java/org/apache/falcon/util/StartupPropertiesTest.java
deleted file mode 100644
index 6b2ec06..0000000
--- a/common/src/test/java/org/apache/falcon/util/StartupPropertiesTest.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.falcon.util;
-
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import static org.testng.AssertJUnit.assertEquals;
-
-/**
- * Test for startup properties test.
- */
-@Test
-public class StartupPropertiesTest {
-    @BeforeClass
-    public void setUp() {
-        StartupProperties.get();
-    }
-
-    public void testDomain() {
-        StartupProperties props = (StartupProperties) StartupProperties.get();
-        assertEquals("debug", props.getDomain());
-        //assertEquals("vm://localhost", props.get("broker.url"));
-        assertEquals("FALCON.ENTITY.TOPIC", props.get("entity.topic"));
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/46dcea2f/common/src/test/resources/classpath.properties
----------------------------------------------------------------------
diff --git a/common/src/test/resources/classpath.properties b/common/src/test/resources/classpath.properties
new file mode 100644
index 0000000..aedc45d
--- /dev/null
+++ b/common/src/test/resources/classpath.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+*.domain=unittest
+
+unittest.test=hello world

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/46dcea2f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ca16762..a7c745e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -478,6 +478,14 @@
                         <groupId>org.eclipse.jdt</groupId>
                         <artifactId>core</artifactId>
                     </exclusion>
+                    <exclusion>
+                        <groupId>tomcat</groupId>
+                        <artifactId>jasper-runtime</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>tomcat</groupId>
+                        <artifactId>jasper-compiler</artifactId>
+                    </exclusion>
                 </exclusions>
             </dependency>
 
@@ -599,6 +607,20 @@
             <resource>
                 <directory>src/main/resources</directory>
                 <filtering>true</filtering>
+                <excludes>
+                    <exclude>startup.properties</exclude>
+                    <exclude>runtime.properties</exclude>
+                    <exclude>log4j.xml</exclude>
+                </excludes>
+            </resource>
+            <resource>
+                <directory>src/main/resources</directory>
+                <filtering>false</filtering>
+                <includes>
+                    <include>startup.properties</include>
+                    <include>runtime.properties</include>
+                    <include>log4j.xml</include>
+                </includes>
             </resource>
         </resources>
         <testResources>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/46dcea2f/prism/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/prism/src/main/resources/log4j.xml b/prism/src/main/resources/log4j.xml
index ef53274..b474d69 100644
--- a/prism/src/main/resources/log4j.xml
+++ b/prism/src/main/resources/log4j.xml
@@ -63,7 +63,7 @@
         </layout>
     </appender>
 
-    <logger name="org.apache.falcon">
+    <logger name="org.apache.falcon" additivity="false">
         <level value="debug"/>
         <appender-ref ref="FILE"/>
     </logger>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/46dcea2f/src/bin/falcon-start
----------------------------------------------------------------------
diff --git a/src/bin/falcon-start b/src/bin/falcon-start
index 9a68220..185aede 100755
--- a/src/bin/falcon-start
+++ b/src/bin/falcon-start
@@ -44,7 +44,7 @@ else
 fi
 
 pushd ${BASEDIR} > /dev/null
-JAVA_PROPERTIES="-Dactivemq.base=${BASEDIR}/logs"
+JAVA_PROPERTIES="-Dactivemq.base=${BASEDIR}/logs -Dconfig.location=${BASEDIR}/conf"
 while [[ ${1} =~ ^\-D ]]; do
   JAVA_PROPERTIES="${JAVA_PROPERTIES} ${1}"
   shift

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/46dcea2f/src/conf/log4j.xml
----------------------------------------------------------------------
diff --git a/src/conf/log4j.xml b/src/conf/log4j.xml
index 738beb5..53b9a96 100644
--- a/src/conf/log4j.xml
+++ b/src/conf/log4j.xml
@@ -61,7 +61,7 @@
         </layout>
     </appender>
 
-    <logger name="org.apache.falcon">
+    <logger name="org.apache.falcon" additivity="false">
         <level value="debug"/>
         <appender-ref ref="FILE"/>
     </logger>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/46dcea2f/src/conf/startup.properties
----------------------------------------------------------------------
diff --git a/src/conf/startup.properties b/src/conf/startup.properties
index a1fdab2..7f1df68 100644
--- a/src/conf/startup.properties
+++ b/src/conf/startup.properties
@@ -47,7 +47,7 @@
 ######### Implementation classes #########
 
 debug.config.store.uri=file://${user.dir}/store
-debug.system.lib.location=${user.dir}/server/lib
+debug.system.lib.location=${user.dir}/server/webapp/lib
 debug.broker.url=tcp://localhost:61616
 debug.retry.recorder.path=${user.dir}/logs/retry
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/46dcea2f/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index 6743527..86242c2 100644
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -173,6 +173,21 @@
             </plugin>
 
             <plugin>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>copy-dependencies</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/dependency</outputDirectory>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-war-plugin</artifactId>
                 <version>2.1.1</version>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/46dcea2f/webapp/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/webapp/src/main/resources/log4j.xml b/webapp/src/main/resources/log4j.xml
index 48fc14b..ed039fe 100644
--- a/webapp/src/main/resources/log4j.xml
+++ b/webapp/src/main/resources/log4j.xml
@@ -63,7 +63,7 @@
         </layout>
     </appender>
 
-    <logger name="org.apache.falcon">
+    <logger name="org.apache.falcon" additivity="false">
         <level value="debug"/>
         <appender-ref ref="FILE"/>
     </logger>


Mime
View raw message