falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkat...@apache.org
Subject [4/8] git commit: FALCON-127 Fix test issues after FALCON-107. Contributed by Shwetha GS
Date Mon, 23 Sep 2013 23:04:52 GMT
FALCON-127 Fix test issues after FALCON-107. Contributed by Shwetha GS


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/518dc895
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/518dc895
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/518dc895

Branch: refs/heads/FALCON-85
Commit: 518dc89528b73d5c65248db56a87518a255446b7
Parents: fa2bc8e
Author: srikanth.sundarrajan <srikanth.sundarrajan@inmobi.com>
Authored: Wed Sep 18 13:53:12 2013 +0530
Committer: srikanth.sundarrajan <srikanth.sundarrajan@inmobi.com>
Committed: Wed Sep 18 13:53:12 2013 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 +++
 docs/src/site/twiki/InstallationSteps.twiki     | 10 +++++---
 .../falcon/converter/OozieFeedMapperTest.java   | 11 +++------
 .../converter/AbstractOozieEntityMapper.java    | 13 ++++++----
 .../converter/OozieProcessMapperTest.java       | 26 +++++++++-----------
 .../falcon/retention/FeedEvictorTest.java       | 16 ++++++------
 6 files changed, 42 insertions(+), 37 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/518dc895/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ddb3418..48e19d0 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -28,6 +28,9 @@ Trunk (Unreleased)
     Srikanth Sundarrajan)
 
   BUG FIXES
+    FALCON-127 Fix test issues after FALCON-107. (Shwetha GS via Srikanth
+    Sundarrajan)
+
     FALCON-76 Use cluster readonly endpoint for replication source.
     (Suhas Vasu via Shweth GS)
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/518dc895/docs/src/site/twiki/InstallationSteps.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki
index c459fb0..ab23d49 100644
--- a/docs/src/site/twiki/InstallationSteps.twiki
+++ b/docs/src/site/twiki/InstallationSteps.twiki
@@ -8,10 +8,10 @@ git clone https://git-wip-us.apache.org/repos/asf/incubator-falcon.git falcon
 
 cd falcon
 
-export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean install
+export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean install [For hadoop
1]
+export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean install -Dhadoop.profile=2
[For hadoop 2]
 
 [optionally -Dhadoop.version=<<hadoop.version>> can be appended to build for
a specific version of hadoop]
-[Falcon has currently not been tested with secure Hadoop / Hadoop 2.0]
 
 </verbatim>
 
@@ -20,7 +20,8 @@ Once the build successfully completes, artifacts can be packaged for deployment.
 *Embedded Mode*
 <verbatim>
 
-mvn clean assembly:assembly -DskipTests -DskipCheck=true
+mvn clean assembly:assembly -DskipTests -DskipCheck=true [For hadoop 1]
+mvn clean assembly:assembly -DskipTests -DskipCheck=true -Dhadoop.profile=2 [For hadoop 2]
 
 </verbatim>
 
@@ -56,7 +57,8 @@ Tar is structured as follows
 
 <verbatim>
 
-mvn clean assembly:assembly -DskipTests -DskipCheck=true -P distributed
+mvn clean assembly:assembly -DskipTests -DskipCheck=true -P distributed -Dhadoop.profile=1
[For hadoop 1]
+mvn clean assembly:assembly -DskipTests -DskipCheck=true -P distributed -Dhadoop.profile=2
[For hadoop 2]
 
 </verbatim>
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/518dc895/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
----------------------------------------------------------------------
diff --git a/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java b/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
index bde6589..4fb6a58 100644
--- a/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
+++ b/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
@@ -19,17 +19,13 @@ package org.apache.falcon.converter;
 
 import static org.testng.Assert.assertEquals;
 
-import java.io.IOException;
-import java.io.StringWriter;
 import java.util.Collection;
 import java.util.List;
 
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.JAXBElement;
-import javax.xml.bind.JAXBException;
 import javax.xml.bind.Unmarshaller;
 
-import org.apache.commons.io.IOUtils;
 import org.apache.falcon.FalconException;
 import org.apache.falcon.cluster.util.EmbeddedCluster;
 import org.apache.falcon.entity.ClusterHelper;
@@ -96,7 +92,8 @@ public class OozieFeedMapperTest {
             ClusterHelper.getInterface(cluster, Interfacetype.WRITE).setEndpoint(writeEndpoint);
             FileSystem fs = new Path(writeEndpoint).getFileSystem(new Configuration());
             fs.create(new Path(ClusterHelper.getLocation(cluster, "working"), "libext/FEED/retention/ext.jar")).close();
-            fs.create(new Path(ClusterHelper.getLocation(cluster, "working"), "libext/FEED/replication/ext.jar")).close();
+            fs.create(
+                    new Path(ClusterHelper.getLocation(cluster, "working"), "libext/FEED/replication/ext.jar")).close();
         }
         return entity;
     }
@@ -173,8 +170,8 @@ public class OozieFeedMapperTest {
 
     private void assertLibExtensions(COORDINATORAPP coord, String lifecycle) throws Exception
{
         String wfPath = coord.getAction().getWorkflow().getAppPath().replace("${nameNode}",
"");
-        JAXBContext WORKFLOW_JAXB_CONTEXT = JAXBContext.newInstance(WORKFLOWAPP.class);
-        WORKFLOWAPP wf = ((JAXBElement<WORKFLOWAPP>) WORKFLOW_JAXB_CONTEXT.createUnmarshaller().unmarshal(
+        JAXBContext jaxbContext = JAXBContext.newInstance(WORKFLOWAPP.class);
+        WORKFLOWAPP wf = ((JAXBElement<WORKFLOWAPP>) jaxbContext.createUnmarshaller().unmarshal(
                 trgMiniDFS.getFileSystem().open(new Path(wfPath, "workflow.xml")))).getValue();
         List<Object> actions = wf.getDecisionOrForkOrJoin();
         for (Object obj : actions) {

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/518dc895/oozie/src/main/java/org/apache/falcon/converter/AbstractOozieEntityMapper.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/converter/AbstractOozieEntityMapper.java
b/oozie/src/main/java/org/apache/falcon/converter/AbstractOozieEntityMapper.java
index 4a55bb6..5ee55c2 100644
--- a/oozie/src/main/java/org/apache/falcon/converter/AbstractOozieEntityMapper.java
+++ b/oozie/src/main/java/org/apache/falcon/converter/AbstractOozieEntityMapper.java
@@ -48,10 +48,7 @@ import org.apache.log4j.Logger;
 import org.apache.oozie.client.OozieClient;
 
 import javax.xml.bind.*;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.StringWriter;
+import java.io.*;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -146,7 +143,13 @@ public abstract class AbstractOozieEntityMapper<T extends Entity>
{
     }
 
     private void addExtensionJars(FileSystem fs, Path path, WORKFLOWAPP wf) throws IOException
{
-        FileStatus[] libs = fs.listStatus(path);
+        FileStatus[] libs = null;
+        try {
+            libs = fs.listStatus(path);
+        } catch(FileNotFoundException ignore) {
+            //Ok if the libext is not configured
+        }
+
         if (libs == null) {
             return;
         }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/518dc895/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
----------------------------------------------------------------------
diff --git a/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
b/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
index 6ed0d9c..9d13d79 100644
--- a/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
+++ b/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
@@ -206,8 +206,8 @@ public class OozieProcessMapperTest extends AbstractTestBase {
 
     private void assertLibExtensions(COORDINATORAPP coord) throws Exception {
         String wfPath = coord.getAction().getWorkflow().getAppPath().replace("${nameNode}",
"");
-        JAXBContext WORKFLOW_JAXB_CONTEXT = JAXBContext.newInstance(WORKFLOWAPP.class);
-        WORKFLOWAPP wf = ((JAXBElement<WORKFLOWAPP>) WORKFLOW_JAXB_CONTEXT.createUnmarshaller().unmarshal(
+        JAXBContext jaxbContext = JAXBContext.newInstance(WORKFLOWAPP.class);
+        WORKFLOWAPP wf = ((JAXBElement<WORKFLOWAPP>) jaxbContext.createUnmarshaller().unmarshal(
                 fs.open(new Path(wfPath, "workflow.xml")))).getValue();
         List<Object> actions = wf.getDecisionOrForkOrJoin();
         for (Object obj : actions) {
@@ -236,24 +236,22 @@ public class OozieProcessMapperTest extends AbstractTestBase {
         OozieProcessMapper mapper = new OozieProcessMapper(process);
         Path bundlePath = new Path("/", EntityUtil.getStagingPath(process));
         mapper.map(cluster, bundlePath);
-
-        FileSystem fs = new Path(hdfsUrl).getFileSystem(new Configuration());
         assertTrue(fs.exists(bundlePath));
 
-        BUNDLEAPP bundle = getBundle(fs, bundlePath);
+        BUNDLEAPP bundle = getBundle(bundlePath);
         assertEquals(EntityUtil.getWorkflowName(process).toString(), bundle.getName());
         assertEquals(1, bundle.getCoordinator().size());
         assertEquals(EntityUtil.getWorkflowName(Tag.DEFAULT, process).toString(),
                 bundle.getCoordinator().get(0).getName());
         String coordPath = bundle.getCoordinator().get(0).getAppPath().replace("${nameNode}",
"");
 
-        COORDINATORAPP coord = getCoordinator(fs, new Path(coordPath));
+        COORDINATORAPP coord = getCoordinator(new Path(coordPath));
         testDefCoordMap(process, coord);
         assertEquals(coord.getControls().getThrottle(), throttle);
         assertEquals(coord.getControls().getTimeout(), timeout);
 
         String wfPath = coord.getAction().getWorkflow().getAppPath().replace("${nameNode}",
"");
-        return getParentWorkflow(fs, new Path(wfPath));
+        return getParentWorkflow(new Path(wfPath));
     }
 
     public void testParentWorkflow(Process process, WORKFLOWAPP parentWorkflow) {
@@ -267,8 +265,8 @@ public class OozieProcessMapperTest extends AbstractTestBase {
         Assert.assertEquals("failed-post-processing", ((ACTION) decisionOrForkOrJoin.get(6)).getName());
     }
 
-    private COORDINATORAPP getCoordinator(FileSystem fs, Path path) throws Exception {
-        String bundleStr = readFile(fs, path);
+    private COORDINATORAPP getCoordinator(Path path) throws Exception {
+        String bundleStr = readFile(path);
 
         Unmarshaller unmarshaller = JAXBContext.newInstance(COORDINATORAPP.class).createUnmarshaller();
         SchemaFactory schemaFactory = SchemaFactory.newInstance("http://www.w3.org/2001/XMLSchema");
@@ -279,8 +277,8 @@ public class OozieProcessMapperTest extends AbstractTestBase {
         return jaxbBundle.getValue();
     }
 
-    private WORKFLOWAPP getParentWorkflow(FileSystem fs, Path path) throws Exception {
-        String workflow = readFile(fs, new Path(path, "workflow.xml"));
+    private WORKFLOWAPP getParentWorkflow(Path path) throws Exception {
+        String workflow = readFile(new Path(path, "workflow.xml"));
 
         Unmarshaller unmarshaller = JAXBContext.newInstance(WORKFLOWAPP.class).createUnmarshaller();
         SchemaFactory schemaFactory = SchemaFactory.newInstance("http://www.w3.org/2001/XMLSchema");
@@ -291,8 +289,8 @@ public class OozieProcessMapperTest extends AbstractTestBase {
         return jaxbWorkflow.getValue();
     }
 
-    private BUNDLEAPP getBundle(FileSystem fs, Path path) throws Exception {
-        String bundleStr = readFile(fs, new Path(path, "bundle.xml"));
+    private BUNDLEAPP getBundle(Path path) throws Exception {
+        String bundleStr = readFile(new Path(path, "bundle.xml"));
 
         Unmarshaller unmarshaller = JAXBContext.newInstance(BUNDLEAPP.class).createUnmarshaller();
         SchemaFactory schemaFactory = SchemaFactory.newInstance("http://www.w3.org/2001/XMLSchema");
@@ -303,7 +301,7 @@ public class OozieProcessMapperTest extends AbstractTestBase {
         return jaxbBundle.getValue();
     }
 
-    private String readFile(FileSystem fs, Path path) throws Exception {
+    private String readFile(Path path) throws Exception {
         BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(path)));
         String line;
         StringBuilder contents = new StringBuilder();

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/518dc895/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
----------------------------------------------------------------------
diff --git a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
index fc55e11..5377c48 100644
--- a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
+++ b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
@@ -52,10 +52,12 @@ public class FeedEvictorTest {
     private EmbeddedCluster cluster;
     private final InMemoryWriter stream = new InMemoryWriter(System.out);
     private final Map<String, String> map = new HashMap<String, String>();
+    private String hdfsUrl;
 
     @BeforeClass
     public void start() throws Exception {
         cluster = EmbeddedCluster.newCluster("test");
+        hdfsUrl = cluster.getConf().get("fs.default.name");
         FeedEvictor.OUT.set(stream);
     }
 
@@ -101,7 +103,7 @@ public class FeedEvictorTest {
             Pair<List<String>, List<String>> pair;
             pair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
             String dataPath = "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
-            String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
+            String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
 
             FeedEvictor.main(new String[]{
                 "-feedBasePath", cluster.getConf().get("fs.default.name") + dataPath,
@@ -183,7 +185,7 @@ public class FeedEvictorTest {
             Pair<List<String>, List<String>> pair;
             pair = createTestData("feed2", "yyyyMMddHH/'more'/yyyy", 5, TimeUnit.HOURS, "/data");
             String dataPath = "/data/YYYY/feed2/mmHH/dd/MM/?{YEAR}?{MONTH}?{DAY}?{HOUR}/more/?{YEAR}";
-            String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
+            String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
             FeedEvictor.main(new String[]{
                 "-feedBasePath", cluster.getConf().get("fs.default.name") + dataPath,
                 "-retentionType", "instance", "-retentionLimit", "hours(5)", "-timeZone",
"UTC", "-frequency",
@@ -214,12 +216,12 @@ public class FeedEvictorTest {
                 "-feedBasePath",
                 cluster.getConf().get("fs.default.name") + "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}",
                 "-retentionType", "instance", "-retentionLimit", "months(5)", "-timeZone",
-                "UTC", "-frequency", "hourly", "-logFile", "/falcon/staging/feed/2012-01-01-04-00",
});
+                "UTC", "-frequency", "hourly", "-logFile", hdfsUrl + "/falcon/staging/feed/2012-01-01-04-00",
});
             Assert.assertEquals("instances=NULL", stream.getBuffer());
 
             stream.clear();
             String dataPath = "/data/YYYY/feed4/dd/MM/02/more/hello";
-            String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
+            String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
             FeedEvictor.main(new String[] {"-feedBasePath",
                 cluster.getConf().get("fs.default.name") + dataPath, "-retentionType", "instance",
                 "-retentionLimit", "hours(5)", "-timeZone", "UTC", "-frequency", "hourly",
"-logFile", logFile, });
@@ -252,12 +254,12 @@ public class FeedEvictorTest {
                     + getFeedBasePath("/meta") + "#"
                     + getFeedBasePath("/tmp"),
                 "-retentionType", "instance", "-retentionLimit", "months(5)", "-timeZone",
-                "UTC", "-frequency", "hourly", "-logFile", "/falcon/staging/feed/2012-01-01-04-00",
});
+                "UTC", "-frequency", "hourly", "-logFile", hdfsUrl + "/falcon/staging/feed/2012-01-01-04-00",
});
             Assert.assertEquals("instances=NULL", stream.getBuffer());
 
             stream.clear();
             String dataPath = "/data/YYYY/feed4/dd/MM/02/more/hello";
-            String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
+            String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
             FeedEvictor.main(new String[]{"-feedBasePath",
                                           cluster.getConf().get("fs.default.name") + dataPath,
                                           "-retentionType", "instance", "-retentionLimit",
@@ -293,7 +295,7 @@ public class FeedEvictorTest {
                     + "#"
                     + cluster.getConf().get("fs.default.name")
                     + "/meta/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
-            String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
+            String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
 
             FeedEvictor.main(new String[] {"-feedBasePath", dataPath, "-retentionType", "instance",
                 "-retentionLimit", "days(10)", "-timeZone", "UTC", "-frequency", "daily",
"-logFile", logFile, });


Mime
View raw message