falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pall...@apache.org
Subject falcon git commit: FALCON-2261 Job submission fails with IllegalArgumentException. Creating falconHdfsFilesystem based on the uri
Date Tue, 24 Jan 2017 11:05:19 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 9da6d3804 -> e132065f1


FALCON-2261 Job submission fails with IllegalArgumentException. Creating falconHdfsFilesystem
based on the uri

Author: sandeep <sandysmdl@gmail.com>

Reviewers: @pallavi-rao

Closes #346 from sandeepSamudrala/FALCON-2261 and squashes the following commits:

3fefbc7 [sandeep] Fixed checkstyle issues
24b3da9 [sandeep] FALCON-2261 Job submission fails with IllegalArgumentException. Creating
falcon hdfs filesystem based on the uri
c5da0a2 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
7e16263 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
a234d94 [sandeep] FALCON-2231 Incoporated review comments and small fixes for duplicate submission
and colo addition to schedule command
26e3350 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
73fbf75 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
cc28658 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
089b10d [sandeep] Merge branch 'master' of https://github.com/apache/falcon
456d4ee [sandeep] Merge branch 'master' of https://github.com/apache/falcon
0cf9af6 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
4a2e23e [sandeep] Merge branch 'master' of https://github.com/apache/falcon
b1546ed [sandeep] Merge branch 'master' of https://github.com/apache/falcon
0a433fb [sandeep] Merge branch 'master' of https://github.com/apache/falcon
194f36a [sandeep] Merge branch 'master' of https://github.com/apache/falcon
e0ad358 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
f96a084 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
9cf36e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
bbca081 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
48f6afa [sandeep] Merge branch 'master' of https://github.com/apache/falcon
250cc46 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
d0393e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
a178805 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
d6dc8bf [sandeep] Merge branch 'master' of https://github.com/apache/falcon
1bb8d3c [sandeep] Merge branch 'master' of https://github.com/apache/falcon
c065566 [sandeep] reverting last line changes made
1a4dcd2 [sandeep] rebased and resolved the conflicts from master
271318b [sandeep] FALCON-2097. Adding UT to the new method for getting next instance time
with Delay.
a94d4fe [sandeep] rebasing from master
9e68a57 [sandeep] FALCON-298. Feed update with replication delay creates holes


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/e132065f
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/e132065f
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/e132065f

Branch: refs/heads/master
Commit: e132065f1e814f86e0dbc21f473e19c695a01a21
Parents: 9da6d38
Author: sandeep <sandysmdl@gmail.com>
Authored: Tue Jan 24 16:35:09 2017 +0530
Committer: Pallavi Rao <pallavi.rao@inmobi.com>
Committed: Tue Jan 24 16:35:09 2017 +0530

----------------------------------------------------------------------
 .../java/org/apache/falcon/ExtensionHandler.java    | 16 +++++++++-------
 .../falcon/extensions/store/ExtensionStore.java     |  5 -----
 .../org/apache/falcon/unit/FalconUnitClient.java    |  3 ++-
 3 files changed, 11 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/e132065f/client/src/main/java/org/apache/falcon/ExtensionHandler.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/ExtensionHandler.java b/client/src/main/java/org/apache/falcon/ExtensionHandler.java
index f6f3346..7d63d11 100644
--- a/client/src/main/java/org/apache/falcon/ExtensionHandler.java
+++ b/client/src/main/java/org/apache/falcon/ExtensionHandler.java
@@ -24,7 +24,7 @@ import org.apache.falcon.client.FalconExtensionConstants;
 import org.apache.falcon.entity.v0.Entity;
 import org.apache.falcon.entity.v0.EntityType;
 import org.apache.falcon.extensions.ExtensionBuilder;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.falcon.hadoop.HadoopClientFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.codehaus.jettison.json.JSONException;
@@ -38,6 +38,8 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLEncoder;
 import java.util.ArrayList;
@@ -95,11 +97,10 @@ public final class ExtensionHandler {
     }
 
     public static List<Entity> loadAndPrepare(String extensionName, String jobName,
InputStream configStream,
-                                              String extensionBuildLocation) throws IOException,
FalconException {
-        Configuration conf = new Configuration();
-        FileSystem fs = FileSystem.get(conf);
+                                              String extensionBuildLocation)
+        throws IOException, FalconException, URISyntaxException {
         String stagePath = createStagePath(extensionName, jobName);
-        List<URL> urls = ExtensionHandler.copyExtensionPackage(extensionBuildLocation,
fs, stagePath);
+        List<URL> urls = ExtensionHandler.copyExtensionPackage(extensionBuildLocation,
stagePath);
 
         List<Entity> entities = prepare(extensionName, jobName, configStream, urls);
         ExtensionHandler.stageEntities(entities, stagePath);
@@ -152,9 +153,10 @@ public final class ExtensionHandler {
         return stagePath;
     }
 
-    private static List<URL> copyExtensionPackage(String extensionBuildUrl, FileSystem
fs, String stagePath)
-        throws IOException {
+    private static List<URL> copyExtensionPackage(String extensionBuildUrl, String
stagePath)
+        throws IOException, FalconException, URISyntaxException {
 
+        FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(new URI(extensionBuildUrl));
         Path libsPath = new Path(extensionBuildUrl, FalconExtensionConstants.LIBS);
         Path buildLibsPath = new Path(libsPath, FalconExtensionConstants.BUILD);
         Path localStagePath = new Path(stagePath);

http://git-wip-us.apache.org/repos/asf/falcon/blob/e132065f/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
index c50d6de..1d71651 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
@@ -29,7 +29,6 @@ import org.apache.falcon.extensions.jdbc.ExtensionMetaStore;
 import org.apache.falcon.hadoop.HadoopClientFactory;
 import org.apache.falcon.persistence.ExtensionBean;
 import org.apache.falcon.util.StartupProperties;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
@@ -300,7 +299,6 @@ public final class ExtensionStore {
     }
 
     private FileSystem getHdfsFileSystem(String path)  throws  FalconException {
-        Configuration conf = new Configuration();
         URI uri;
         try {
             uri = new URI(path);
@@ -308,19 +306,16 @@ public final class ExtensionStore {
             LOG.error("Exception : ", e);
             throw new FalconException(e);
         }
-        conf.set("fs.default.name", uri.getScheme() + "://" + uri.getAuthority());
         return HadoopClientFactory.get().createFalconFileSystem(uri);
     }
 
 
     public String registerExtension(final String extensionName, final String path, final
String description,
                                     String extensionOwner) throws URISyntaxException, FalconException
{
-        Configuration conf = new Configuration();
         URI uri = new URI(path);
         assertURI("Scheme", uri.getScheme());
         assertURI("Authority", uri.getAuthority());
         assertURI("Path", uri.getPath());
-        conf.set("fs.defaultFS", uri.getScheme() + "://" + uri.getAuthority());
         FileSystem fileSystem = getHdfsFileSystem(path);
         try {
             fileSystem.listStatus(new Path(uri.getPath() + "/README"));

http://git-wip-us.apache.org/repos/asf/falcon/blob/e132065f/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java
----------------------------------------------------------------------
diff --git a/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java b/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java
index 9358958..75aeba0 100644
--- a/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java
+++ b/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java
@@ -56,6 +56,7 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
 import java.util.TreeMap;
 import java.util.ArrayList;
 import java.util.Properties;
@@ -330,7 +331,7 @@ public class FalconUnitClient extends AbstractFalconClient {
         try {
             entities = ExtensionHandler.loadAndPrepare(extensionName, jobName, configStream,
                     packagePath);
-        } catch (FalconException | IOException e) {
+        } catch (FalconException | IOException | URISyntaxException e) {
             throw new FalconCLIException("Failed in generating entities" + jobName);
         }
         return entities;


Mime
View raw message