falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shweth...@apache.org
Subject git commit: FALCON-260 When a process is scheduled, the user workflow is failing with OozieClientException. Contributed by Shwetha GS
Date Mon, 20 Jan 2014 11:49:15 GMT
Updated Branches:
  refs/heads/master 81051f300 -> d555dd501


FALCON-260 When a process is scheduled, the user workflow is failing with OozieClientException.
Contributed by Shwetha GS


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/d555dd50
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/d555dd50
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/d555dd50

Branch: refs/heads/master
Commit: d555dd50136e2dd0c76ccad407d56e88a03810fd
Parents: 81051f3
Author: Shwetha GS <shwethags@gmail.com>
Authored: Mon Jan 20 17:19:08 2014 +0530
Committer: Shwetha GS <shwethags@gmail.com>
Committed: Mon Jan 20 17:19:08 2014 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 +++
 .../org/apache/falcon/update/UpdateHelper.java  | 22 ++++++++++++++------
 .../apache/falcon/update/UpdateHelperTest.java  | 21 +++++++++++++++++++
 src/bin/falcon                                  |  7 +------
 4 files changed, 41 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d555dd50/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 67dba7e..efe0d88 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -31,6 +31,9 @@ Trunk (Unreleased)
   OPTIMIZATIONS
 
   BUG FIXES
+    FALCON-260 When a process is scheduled, the user workflow is failing with OozieClientException.
+    (Shwetha GS)
+
     FALCON-268 Checkstyle/Findbugs issues on FalconCLI. (Jean-Baptiste Onofré via Shwetha
GS)
 
     FALCON-258 Falcon status throws an error when external jobids are missing (Suhas Vasu
via Shaik Idris)

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d555dd50/common/src/main/java/org/apache/falcon/update/UpdateHelper.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/update/UpdateHelper.java b/common/src/main/java/org/apache/falcon/update/UpdateHelper.java
index 84501de..7ed4394 100644
--- a/common/src/main/java/org/apache/falcon/update/UpdateHelper.java
+++ b/common/src/main/java/org/apache/falcon/update/UpdateHelper.java
@@ -136,16 +136,22 @@ public final class UpdateHelper {
         }
     }
 
-    //Recursively traverses each file and tracks checksum
-    //If dest != null, each traversed file is copied to dest
+    /**
+     * Recursively traverses each file and tracks checksum. If dest != null, each traversed
file is copied to dest
+     * @param fs FileSystem
+     * @param src file/directory
+     * @param dest directory always
+     * @return checksums
+     * @throws FalconException
+     */
     public static Map<String, String> checksumAndCopy(FileSystem fs, Path src, Path
dest) throws FalconException {
         try {
+            Configuration conf = new Configuration();
+            Map<String, String> paths = new HashMap<String, String>();
             if (dest != null && !fs.exists(dest) && !fs.mkdirs(dest)) {
                 throw new FalconException("mkdir failed on " + dest);
             }
 
-            Configuration conf = new Configuration();
-            Map<String, String> paths = new HashMap<String, String>();
             if (fs.isFile(src)) {
                 paths.put(src.toString(), fs.getFileChecksum(src).toString());
                 if (dest != null) {
@@ -157,8 +163,12 @@ public final class UpdateHelper {
                 FileStatus[] files = fs.listStatus(src);
                 if (files != null) {
                     for (FileStatus file : files) {
-                        paths.putAll(checksumAndCopy(fs, file.getPath(),
-                                ((dest == null) ? null : new Path(dest, file.getPath().getName()))));
+                        if (fs.isFile(file.getPath())) {
+                            paths.putAll(checksumAndCopy(fs, file.getPath(), dest));
+                        } else {
+                            paths.putAll(checksumAndCopy(fs, file.getPath(),
+                                    ((dest == null) ? null : new Path(dest, file.getPath().getName()))));
+                        }
                     }
                 }
             }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d555dd50/common/src/test/java/org/apache/falcon/update/UpdateHelperTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/update/UpdateHelperTest.java b/common/src/test/java/org/apache/falcon/update/UpdateHelperTest.java
index 7718109..43dcf4d 100644
--- a/common/src/test/java/org/apache/falcon/update/UpdateHelperTest.java
+++ b/common/src/test/java/org/apache/falcon/update/UpdateHelperTest.java
@@ -94,6 +94,27 @@ public class UpdateHelperTest extends AbstractTestBase {
     }
 
     @Test
+    public void testCopyUserWorkflow() throws Exception {
+        FileSystem fs = dfsCluster.getFileSystem();
+        Path basePath = new Path("/tmp/basepath");
+        fs.mkdirs(basePath);
+        Path wfdir = new Path(basePath, "workflow");
+        fs.mkdirs(wfdir);
+        Path wf = new Path(wfdir, "workflow.xml");
+        Path lib = new Path(wfdir, "common.jar");
+        fs.create(wf).close();
+        fs.create(lib).close();
+        Path dest = new Path("/tmp/dest");
+        UpdateHelper.checksumAndCopy(fs, wfdir, dest);
+        Assert.assertTrue(fs.exists(new Path(dest, "workflow.xml")) && fs.isFile(new
Path(dest, "workflow.xml")));
+        Assert.assertTrue(fs.exists(new Path(dest, "common.jar")) && fs.isFile(new
Path(dest, "common.jar")));
+
+        fs.delete(dest, true);
+        UpdateHelper.checksumAndCopy(fs, wf, dest);
+        Assert.assertTrue(fs.exists(new Path(dest, "workflow.xml")) && fs.isFile(new
Path(dest, "workflow.xml")));
+    }
+
+    @Test
     public void testWorkflowUpdate() throws IOException, FalconException {
         FileSystem fs = dfsCluster.getFileSystem();
         Process process = processParser.parseAndValidate(this.getClass().getResourceAsStream(PROCESS_XML));

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d555dd50/src/bin/falcon
----------------------------------------------------------------------
diff --git a/src/bin/falcon b/src/bin/falcon
index 08478d7..d196a5d 100755
--- a/src/bin/falcon
+++ b/src/bin/falcon
@@ -30,9 +30,4 @@ BASEDIR=`dirname ${PRG}`
 BASEDIR=`cd ${BASEDIR}/..;pwd`
 . ${BASEDIR}/bin/falcon-config.sh 'client'
 
-JAVA_PROPERTIES="$FALCON_OPTS"
-while [[ ${1} =~ ^\-D ]]; do
-  JAVA_PROPERTIES="${JAVA_PROPERTIES} ${1}"
-  shift
-done
-${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${FALCONCPPATH} org.apache.falcon.cli.FalconCLI "${@}"
\ No newline at end of file
+${JAVA_BIN} -cp ${FALCONCPPATH} org.apache.falcon.cli.FalconCLI "${@}"


Mime
View raw message