falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkat...@apache.org
Subject git commit: FALCON-32 Store test output in file rather then printing all the output into the console. Contributed by Srikanth Sundarrajan
Date Wed, 10 Jul 2013 20:32:01 GMT
Updated Branches:
  refs/heads/master 614173574 -> d5aa7a92b


FALCON-32 Store test output in file rather then printing all the output into the console.
Contributed by Srikanth Sundarrajan


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/d5aa7a92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/d5aa7a92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/d5aa7a92

Branch: refs/heads/master
Commit: d5aa7a92b8a1d7f398d5abaa8b681b90e8551ccd
Parents: 6141735
Author: venkatesh <venkatesh@hortonworks.com>
Authored: Wed Jul 10 13:32:49 2013 -0700
Committer: venkatesh <venkatesh@hortonworks.com>
Committed: Wed Jul 10 13:32:49 2013 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  8 +++-
 .../apache/falcon/entity/AbstractTestBase.java  | 22 ++++++++--
 .../falcon/entity/ColoClusterRelationTest.java  | 13 +++---
 .../apache/falcon/group/FeedGroupMapTest.java   | 22 +++++-----
 hadoop-webapp/pom.xml                           |  1 +
 hadoop-webapp/src/main/resources/log4j.xml      | 38 +++++++++++++++++
 .../src/main/resources/oozie-log4j.properties   | 43 ++++++++++++++++++++
 pom.xml                                         |  3 ++
 webapp/pom.xml                                  |  8 +++-
 webapp/src/main/resources/log4j.xml             |  2 +-
 10 files changed, 134 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d5aa7a92/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
old mode 100644
new mode 100755
index 9f07f31..1d3fed5
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -8,7 +8,13 @@ Trunk (Unreleased)
 
   IMPROVEMENTS
 
-    FALCON-29 Add ability to tag/classify data sets and processes to enable 
+    FALCON-32 Store test output in file rather then printing all the output
+    into the console (Srikanth Sundarrajan via Venkatesh Seetharam)
+
+    FALCON-1 Create packaging and scripts to install and try Apache Falcon
+    (Srikanth Sundarrajan via Venkatesh Seetharam)
+
+    FALCON-29 Add ability to tag/classify data sets and processes to enable
     discovery (Venkatesh Seetharam via Srikanth Sundarrajan)
 
     FALCON-31 File Installation-steps.txt contains old path to repository (Jarek 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d5aa7a92/common/src/test/java/org/apache/falcon/entity/AbstractTestBase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/AbstractTestBase.java b/common/src/test/java/org/apache/falcon/entity/AbstractTestBase.java
index f08c6ac..7668c7f 100644
--- a/common/src/test/java/org/apache/falcon/entity/AbstractTestBase.java
+++ b/common/src/test/java/org/apache/falcon/entity/AbstractTestBase.java
@@ -18,6 +18,7 @@
 
 package org.apache.falcon.entity;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.falcon.FalconException;
 import org.apache.falcon.cluster.util.EmbeddedCluster;
 import org.apache.falcon.entity.store.ConfigurationStore;
@@ -37,7 +38,9 @@ import org.testng.annotations.BeforeClass;
 import javax.xml.bind.JAXBException;
 import javax.xml.bind.Marshaller;
 import javax.xml.bind.Unmarshaller;
+import java.io.File;
 import java.io.StringWriter;
+import java.net.URI;
 import java.util.Collection;
 
 /**
@@ -49,18 +52,29 @@ public class AbstractTestBase {
     protected static final String CLUSTER_XML = "/config/cluster/cluster-0.1.xml";
     protected EmbeddedCluster dfsCluster;
     protected Configuration conf = new Configuration();
+    private ConfigurationStore store;
+
+    public ConfigurationStore getStore() {
+        return store;
+    }
 
     @BeforeClass
     public void initConfigStore() throws Exception {
+        String configPath = new URI(StartupProperties.get().getProperty("config.store.uri")).getPath();
+        String location = configPath + "-" + getClass().getName();
+        StartupProperties.get().setProperty("config.store.uri", location);
+        FileUtils.deleteDirectory(new File(location));
+
         cleanupStore();
         String listeners = StartupProperties.get().getProperty("configstore.listeners");
         StartupProperties.get().setProperty("configstore.listeners",
                 listeners.replace("org.apache.falcon.service.SharedLibraryHostingService",
""));
-        ConfigurationStore.get().init();
+        store = ConfigurationStore.get();
+        store.init();
     }
 
     protected void cleanupStore() throws FalconException {
-        ConfigurationStore store = ConfigurationStore.get();
+        store = ConfigurationStore.get();
         for (EntityType type : EntityType.values()) {
             Collection<String> entities = store.getEntities(type);
             for (String entity : entities) {
@@ -71,7 +85,7 @@ public class AbstractTestBase {
 
     protected void storeEntity(EntityType type, String name) throws Exception {
         Unmarshaller unmarshaller = type.getUnmarshaller();
-        ConfigurationStore store = ConfigurationStore.get();
+        store = ConfigurationStore.get();
         store.remove(type, name);
         switch (type) {
         case CLUSTER:
@@ -102,7 +116,7 @@ public class AbstractTestBase {
     }
 
     public void setup() throws Exception {
-        ConfigurationStore store = ConfigurationStore.get();
+        store = ConfigurationStore.get();
         for (EntityType type : EntityType.values()) {
             for (String name : store.getEntities(type)) {
                 store.remove(type, name);

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d5aa7a92/common/src/test/java/org/apache/falcon/entity/ColoClusterRelationTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/ColoClusterRelationTest.java b/common/src/test/java/org/apache/falcon/entity/ColoClusterRelationTest.java
index 936b478..0d6e754 100644
--- a/common/src/test/java/org/apache/falcon/entity/ColoClusterRelationTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/ColoClusterRelationTest.java
@@ -18,7 +18,6 @@
 
 package org.apache.falcon.entity;
 
-import org.apache.falcon.entity.store.ConfigurationStore;
 import org.apache.falcon.entity.v0.EntityType;
 import org.apache.falcon.entity.v0.cluster.Cluster;
 import org.testng.Assert;
@@ -31,6 +30,7 @@ import java.util.Set;
  */
 @Test
 public class ColoClusterRelationTest extends AbstractTestBase {
+
     private Cluster newCluster(String name, String colo) {
         Cluster cluster = new Cluster();
         cluster.setName(name);
@@ -43,10 +43,9 @@ public class ColoClusterRelationTest extends AbstractTestBase {
         Cluster cluster1 = newCluster("cluster1", "colo1");
         Cluster cluster2 = newCluster("cluster2", "colo1");
         Cluster cluster3 = newCluster("cluster3", "colo2");
-        ConfigurationStore store = ConfigurationStore.get();
-        store.publish(EntityType.CLUSTER, cluster1);
-        store.publish(EntityType.CLUSTER, cluster2);
-        store.publish(EntityType.CLUSTER, cluster3);
+        getStore().publish(EntityType.CLUSTER, cluster1);
+        getStore().publish(EntityType.CLUSTER, cluster2);
+        getStore().publish(EntityType.CLUSTER, cluster3);
 
         ColoClusterRelation relation = ColoClusterRelation.get();
         Set<String> clusters = relation.getClusters("colo1");
@@ -60,13 +59,13 @@ public class ColoClusterRelationTest extends AbstractTestBase {
         Assert.assertEquals(1, clusters.size());
         Assert.assertTrue(clusters.contains(cluster3.getName()));
 
-        store.remove(EntityType.CLUSTER, cluster1.getName());
+        getStore().remove(EntityType.CLUSTER, cluster1.getName());
         clusters = relation.getClusters("colo1");
         Assert.assertNotNull(clusters);
         Assert.assertEquals(1, clusters.size());
         Assert.assertTrue(clusters.contains(cluster2.getName()));
 
-        store.remove(EntityType.CLUSTER, cluster2.getName());
+        getStore().remove(EntityType.CLUSTER, cluster2.getName());
         clusters = relation.getClusters("colo1");
         Assert.assertNotNull(clusters);
         Assert.assertEquals(0, clusters.size());

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d5aa7a92/common/src/test/java/org/apache/falcon/group/FeedGroupMapTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/group/FeedGroupMapTest.java b/common/src/test/java/org/apache/falcon/group/FeedGroupMapTest.java
index a2087c0..cded90b 100644
--- a/common/src/test/java/org/apache/falcon/group/FeedGroupMapTest.java
+++ b/common/src/test/java/org/apache/falcon/group/FeedGroupMapTest.java
@@ -20,7 +20,6 @@ package org.apache.falcon.group;
 import junit.framework.Assert;
 import org.apache.falcon.FalconException;
 import org.apache.falcon.entity.AbstractTestBase;
-import org.apache.falcon.entity.store.ConfigurationStore;
 import org.apache.falcon.entity.v0.EntityType;
 import org.apache.falcon.entity.v0.cluster.Cluster;
 import org.apache.falcon.entity.v0.feed.Feed;
@@ -38,7 +37,6 @@ import java.util.Map;
  * Feed group map tests.
  */
 public class FeedGroupMapTest extends AbstractTestBase {
-    private ConfigurationStore store = ConfigurationStore.get();
     private static Cluster cluster;
 
     @BeforeClass
@@ -57,7 +55,7 @@ public class FeedGroupMapTest extends AbstractTestBase {
 
     @Test
     public void testOnAdd() throws FalconException, JAXBException {
-        store.publish(EntityType.CLUSTER, cluster);
+        getStore().publish(EntityType.CLUSTER, cluster);
         Feed feed1 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
                 FeedGroupMapTest.class
                         .getResourceAsStream("/config/feed/feed-0.1.xml"));
@@ -68,7 +66,7 @@ public class FeedGroupMapTest extends AbstractTestBase {
         location.setType(LocationType.DATA);
         feed1.setLocations(new Locations());
         feed1.getLocations().getLocations().add(location);
-        store.publish(EntityType.FEED, feed1);
+        getStore().publish(EntityType.FEED, feed1);
         Map<String, FeedGroup> groupMapping = FeedGroupMap.get()
                 .getGroupsMapping();
 
@@ -97,7 +95,7 @@ public class FeedGroupMapTest extends AbstractTestBase {
         location.setType(LocationType.DATA);
         feed2.setLocations(new Locations());
         feed2.getLocations().getLocations().add(location);
-        store.publish(EntityType.FEED, feed2);
+        getStore().publish(EntityType.FEED, feed2);
         groupMapping = FeedGroupMap.get().getGroupsMapping();
 
         group = groupMapping.get("group1");
@@ -128,14 +126,14 @@ public class FeedGroupMapTest extends AbstractTestBase {
                 FeedGroupMapTest.class
                         .getResourceAsStream("/config/feed/feed-0.1.xml"));
         feed1.setName("f1");
-        store.publish(EntityType.CLUSTER, cluster);
+        getStore().publish(EntityType.CLUSTER, cluster);
         feed1.setGroups("group7,group8,group9");
         Location location = new Location();
         location.setPath("/projects/bi/rmc/daily/ad/${YEAR}/fraud/${MONTH}-${DAY}/ad");
         location.setType(LocationType.DATA);
         feed1.setLocations(new Locations());
         feed1.getLocations().getLocations().add(location);
-        store.publish(EntityType.FEED, feed1);
+        getStore().publish(EntityType.FEED, feed1);
 
         Feed feed2 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
                 FeedGroupMapTest.class
@@ -146,12 +144,12 @@ public class FeedGroupMapTest extends AbstractTestBase {
         location.setType(LocationType.DATA);
         feed2.setLocations(new Locations());
         feed2.getLocations().getLocations().add(location);
-        store.publish(EntityType.FEED, feed2);
+        getStore().publish(EntityType.FEED, feed2);
 
         Map<String, FeedGroup> groupMapping = FeedGroupMap.get()
                 .getGroupsMapping();
 
-        store.remove(EntityType.FEED, "f2");
+        getStore().remove(EntityType.FEED, "f2");
 
         FeedGroup group = groupMapping.get("group7");
         Assert.assertEquals(group.getName(), "group7");
@@ -164,7 +162,7 @@ public class FeedGroupMapTest extends AbstractTestBase {
         group = groupMapping.get("group10");
         Assert.assertEquals(null, group);
 
-        store.remove(EntityType.FEED, "f1");
+        getStore().remove(EntityType.FEED, "f1");
 
         group = groupMapping.get("group7");
         Assert.assertEquals(null, group);
@@ -183,14 +181,14 @@ public class FeedGroupMapTest extends AbstractTestBase {
                 FeedGroupMapTest.class
                         .getResourceAsStream("/config/feed/feed-0.1.xml"));
         feed1.setName("f5" + System.currentTimeMillis());
-        store.publish(EntityType.CLUSTER, cluster);
+        getStore().publish(EntityType.CLUSTER, cluster);
         feed1.setGroups(null);
         Location location = new Location();
         location.setPath("/projects/bi/rmc/daily/ad/${YEAR}/fraud/${MONTH}-${DAY}/ad");
         location.setType(LocationType.DATA);
         feed1.setLocations(new Locations());
         feed1.getLocations().getLocations().add(location);
-        store.publish(EntityType.FEED, feed1);
+        getStore().publish(EntityType.FEED, feed1);
 
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d5aa7a92/hadoop-webapp/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/pom.xml b/hadoop-webapp/pom.xml
index ba5ee02..d709bab 100644
--- a/hadoop-webapp/pom.xml
+++ b/hadoop-webapp/pom.xml
@@ -95,6 +95,7 @@
                             <target>
                                 <war destfile="target/oozie.war" webxml="target/oozie/WEB-INF/web.xml">
                                     <lib dir="target/oozie/WEB-INF/lib" />
+                                    <classes dir="target/classes" />
                                 </war>
                             </target>
                         </configuration>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d5aa7a92/hadoop-webapp/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/resources/log4j.xml b/hadoop-webapp/src/main/resources/log4j.xml
new file mode 100644
index 0000000..97ef239
--- /dev/null
+++ b/hadoop-webapp/src/main/resources/log4j.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+    <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+        <param name="File" value="${user.dir}/logs/jetty.log"/>
+        <param name="Append" value="true"/>
+        <param name="Threshold" value="debug"/>
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
+        </layout>
+    </appender>
+
+    <root>
+        <priority value="info"/>
+        <appender-ref ref="FILE"/>
+    </root>
+
+</log4j:configuration>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d5aa7a92/hadoop-webapp/src/main/resources/oozie-log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/resources/oozie-log4j.properties b/hadoop-webapp/src/main/resources/oozie-log4j.properties
new file mode 100644
index 0000000..34e1634
--- /dev/null
+++ b/hadoop-webapp/src/main/resources/oozie-log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+# If the Java System property 'oozie.log.dir' is not defined at Oozie start up time
+# XLogService sets its value to '${oozie.home}/logs'
+
+log4j.appender.oozie=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.oozie.DatePattern='.'yyyy-MM-dd-HH
+
+log4j.appender.oozie.File=${oozie.log.dir}/oozie.log
+log4j.appender.oozie.Append=true
+log4j.appender.oozie.layout=org.apache.log4j.PatternLayout
+log4j.appender.oozie.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n
+
+log4j.logger.org.apache.oozie=DEBUG, oozie
+log4j.logger.org.apache.hadoop=WARN, oozie
+log4j.logger.org.mortbay=WARN, oozie
+log4j.logger.org.hsqldb=WARN, oozie
+log4j.logger.org.apache.hadoop.security.authentication.server=DEBUG, oozie

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d5aa7a92/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 854210b..facf090 100644
--- a/pom.xml
+++ b/pom.xml
@@ -772,6 +772,8 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
                 <configuration>
+                    <redirectTestOutputToFile>true</redirectTestOutputToFile>
+                    <forkMode>always</forkMode>
                     <argLine>-Djava.security.krb5.realm= -Djava.security.krb5.kdc=</argLine>
                 </configuration>
             </plugin>
@@ -789,6 +791,7 @@
                     </execution>
                 </executions>
             </plugin>
+
             <plugin>
                 <groupId>org.apache.rat</groupId>
                 <artifactId>apache-rat-plugin</artifactId>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d5aa7a92/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index f73470e..e9c1c8f 100644
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -194,7 +194,7 @@
                 <executions>
                     <execution>
                         <id>copy-dependencies</id>
-                        <phase>process-resources</phase>
+                        <phase>prepare-package</phase>
                         <goals>
                             <goal>copy</goal>
                         </goals>
@@ -223,6 +223,8 @@
                     <execution>
                         <configuration>
                             <target>
+                                <delete dir="../target/store" />
+                                <delete dir="target/webapps/oozie/data" />
                                 <copy file="../hadoop-webapp/target/oozie.war" toDir="target/webapps"
/>
                                 <delete dir="../target/store" />
                             </target>
@@ -239,6 +241,10 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-failsafe-plugin</artifactId>
                 <version>2.15</version>
+                <configuration>
+                    <redirectTestOutputToFile>true</redirectTestOutputToFile>
+                    <forkMode>always</forkMode>
+                </configuration>
                 <executions>
                     <execution>
                         <id>integration-test</id>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d5aa7a92/webapp/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/webapp/src/main/resources/log4j.xml b/webapp/src/main/resources/log4j.xml
index 4183181..48fc14b 100644
--- a/webapp/src/main/resources/log4j.xml
+++ b/webapp/src/main/resources/log4j.xml
@@ -85,7 +85,7 @@
 
     <root>
         <priority value="info"/>
-        <appender-ref ref="console"/>
+        <appender-ref ref="FILE"/>
     </root>
 
 </log4j:configuration>


Mime
View raw message