falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkat...@apache.org
Subject [2/2] git commit: FALCON-257 File system storage wont work with relative paths. Contributed by Venkatesh Seetharam
Date Tue, 11 Mar 2014 00:07:33 GMT
FALCON-257 File system storage wont work with relative paths. Contributed by Venkatesh Seetharam


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/3158c032
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/3158c032
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/3158c032

Branch: refs/heads/master
Commit: 3158c0321d86c474a6edc117a008fc518215663e
Parents: 5445e10
Author: Venkatesh Seetharam <venkatesh@hortonworks.com>
Authored: Mon Mar 10 10:50:22 2014 -0700
Committer: Venkatesh Seetharam <venkatesh@hortonworks.com>
Committed: Mon Mar 10 17:07:37 2014 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |   3 +
 .../apache/falcon/entity/FileSystemStorage.java |  26 ++++-
 .../falcon/entity/FileSystemStorageTest.java    | 105 +++++++++++++++++++
 3 files changed, 133 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3158c032/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b17437b..64bb353 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -66,6 +66,9 @@ Trunk (Unreleased)
 
     FALCON-287 Record lineage information in post processing (Venkatesh Seetharam)
 
+    FALCON-257 File system storage wont work with relative paths
+    (Venkatesh Seetharam)
+
   OPTIMIZATIONS
     FALCON-123 Improve build speeds in falcon. (Srikanth Sundarrajan via Shwetha GS)
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3158c032/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
index 4bcf271..c2b86b6 100644
--- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
@@ -23,6 +23,7 @@ import org.apache.falcon.entity.v0.feed.Feed;
 import org.apache.falcon.entity.v0.feed.Location;
 import org.apache.falcon.entity.v0.feed.LocationType;
 import org.apache.falcon.entity.v0.feed.Locations;
+import org.apache.falcon.security.CurrentUser;
 import org.apache.hadoop.fs.Path;
 
 import java.net.URI;
@@ -166,7 +167,30 @@ public class FileSystemStorage implements Storage {
         }
 
         // normalize the path so trailing and double '/' are removed
-        return storageUrl + new Path(locationForType.getPath());
+        Path locationPath = new Path(locationForType.getPath());
+        locationPath = locationPath.makeQualified(getDefaultUri(), getWorkingDir());
+
+        if (isRelativePath(locationPath)) {
+            locationPath = new Path(storageUrl + locationPath);
+        }
+
+        return locationPath.toString();
+    }
+
+    private boolean isRelativePath(Path locationPath) {
+        return locationPath.toUri().getAuthority() == null && isStorageUrlATemplate();
+    }
+
+    private boolean isStorageUrlATemplate() {
+        return storageUrl.startsWith(FILE_SYSTEM_URL);
+    }
+
+    private URI getDefaultUri() {
+        return new Path(isStorageUrlATemplate() ? "/" : storageUrl).toUri();
+    }
+
+    public Path getWorkingDir() {
+        return new Path(CurrentUser.getSubject() == null ? "/" : "/user/" + CurrentUser.getUser());
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3158c032/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java b/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
index 1d56a78..e095f8e 100644
--- a/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
@@ -20,7 +20,9 @@ package org.apache.falcon.entity;
 
 import org.apache.falcon.entity.v0.feed.Location;
 import org.apache.falcon.entity.v0.feed.LocationType;
+import org.apache.falcon.security.CurrentUser;
 import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
@@ -32,6 +34,13 @@ import java.util.List;
  */
 public class FileSystemStorageTest {
 
+    private static final String USER = "falcon";
+
+    @BeforeClass
+    public void setUp() {
+        CurrentUser.authenticate(USER);
+    }
+
     @Test
     public void testGetType() throws Exception {
         final Location location = new Location();
@@ -80,6 +89,102 @@ public class FileSystemStorageTest {
     }
 
     @Test
+    public void testFSHomeDir() {
+        final Location location = new Location();
+        location.setPath("foo/bar"); // relative path
+        location.setType(LocationType.DATA);
+        List<Location> locations = new ArrayList<Location>();
+        locations.add(location);
+
+        FileSystemStorage storage = new FileSystemStorage("hdfs://localhost:41020", locations);
+        Assert.assertEquals(storage.getWorkingDir().toString(), "/user/falcon");
+    }
+
+    @Test
+    public void testGetUriTemplateForDataWithRelativePath() throws Exception {
+        final Location location = new Location();
+        location.setPath("foo/bar"); // relative path
+        location.setType(LocationType.DATA);
+        List<Location> locations = new ArrayList<Location>();
+        locations.add(location);
+
+        FileSystemStorage storage = new FileSystemStorage("hdfs://localhost:41020", locations);
+        Assert.assertEquals(storage.getUriTemplate(LocationType.DATA),
+                "hdfs://localhost:41020/user/" + USER + "/foo/bar");
+
+        storage = new FileSystemStorage("hdfs://localhost:41020/", locations);
+        Assert.assertEquals(storage.getUriTemplate(LocationType.DATA),
+                "hdfs://localhost:41020/user/" + USER + "/foo/bar");
+    }
+
+    @Test
+    public void testGetUriTemplateForDataWithAbsolutePath() throws Exception {
+        final Location location = new Location();
+        location.setPath("/foo/bar"); // absolute path
+        location.setType(LocationType.DATA);
+        List<Location> locations = new ArrayList<Location>();
+        locations.add(location);
+
+        FileSystemStorage storage = new FileSystemStorage("hdfs://localhost:41020", locations);
+        Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), "hdfs://localhost:41020/foo/bar");
+
+        storage = new FileSystemStorage("hdfs://localhost:41020/", locations);
+        Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), "hdfs://localhost:41020/foo/bar");
+    }
+
+    @Test
+    public void testGetUriTemplateForDataWithAbsoluteURL() throws Exception {
+        final String absoluteUrl = "s3://host:1000/foo/bar";
+        final Location location = new Location();
+        location.setPath(absoluteUrl); // absolute url
+        location.setType(LocationType.DATA);
+        List<Location> locations = new ArrayList<Location>();
+        locations.add(location);
+
+        FileSystemStorage storage = new FileSystemStorage("hdfs://localhost:41020", locations);
+        Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), absoluteUrl);
+
+        storage = new FileSystemStorage("hdfs://localhost:41020/", locations);
+        Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), absoluteUrl);
+    }
+
+    @DataProvider(name = "locationTestWithRelativePathDataProvider")
+    private Object[][] createLocationTestDataWithRelativePath() {
+        return new Object[][] {
+            {"hdfs://h:0", "localDC/rc/billing/ua2", "hdfs://h:0/user/" + USER + "/localDC/rc/billing/ua2"},
+            {"hdfs://h:0/", "localDC/rc/billing/ua2", "hdfs://h:0/user/" + USER + "/localDC/rc/billing/ua2"},
+            {"hdfs://h:0", "localDC/rc/billing/ua2/", "hdfs://h:0/user/" + USER + "/localDC/rc/billing/ua2"},
+            {"hdfs://h:0/", "localDC/rc/billing/ua2/", "hdfs://h:0/user/" + USER + "/localDC/rc/billing/ua2"},
+            {"hdfs://h:0", "localDC/rc/billing/ua2//", "hdfs://h:0/user/" + USER + "/localDC/rc/billing/ua2"},
+            {"hdfs://h:0/", "localDC/rc/billing/ua2//", "hdfs://h:0/user/" + USER + "/localDC/rc/billing/ua2"},
+            {"${nameNode}", "localDC/rc/billing/ua2", "${nameNode}/user/" + USER + "/localDC/rc/billing/ua2"},
+            {"${nameNode}/", "localDC/rc/billing/ua2", "${nameNode}/user/" + USER + "/localDC/rc/billing/ua2"},
+            {"${nameNode}", "localDC/rc/billing/ua2/", "${nameNode}/user/" + USER + "/localDC/rc/billing/ua2"},
+            {"${nameNode}/", "localDC/rc/billing/ua2/", "${nameNode}/user/" + USER + "/localDC/rc/billing/ua2"},
+            {"${nameNode}", "localDC/rc/billing/ua2//", "${nameNode}/user/" + USER + "/localDC/rc/billing/ua2"},
+            {"${nameNode}/", "localDC/rc/billing/ua2//", "${nameNode}/user/" + USER + "/localDC/rc/billing/ua2"},
+            {"${nameNode}/", "localDC/rc/billing/ua2//", "${nameNode}/user/" + USER + "/localDC/rc/billing/ua2"},
+            {"${nameNode}", "s3://h:p/localDC/rc/billing/ua2//", "s3://h:p/localDC/rc/billing/ua2"},
+            {"${nameNode}/", "s3://h:p/localDC/rc/billing/ua2//", "s3://h:p/localDC/rc/billing/ua2"},
+            {"hdfs://h:0", "s3://h:p/localDC/rc/billing/ua2//", "s3://h:p/localDC/rc/billing/ua2"},
+            {"hdfs://h:0/", "s3://h:p/localDC/rc/billing/ua2//", "s3://h:p/localDC/rc/billing/ua2"},
+        };
+    }
+
+    @Test (dataProvider = "locationTestWithRelativePathDataProvider")
+    public void testGetUriTemplateWithRelativePath(String storageUrl, String path,
+                                                   String expected) throws Exception {
+        final Location location = new Location();
+        location.setPath(path);
+        location.setType(LocationType.DATA);
+        List<Location> locations = new ArrayList<Location>();
+        locations.add(location);
+
+        FileSystemStorage storage = new FileSystemStorage(storageUrl, locations);
+        Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), expected);
+    }
+
+    @Test
     public void testGetUriTemplate() throws Exception {
         final Location dataLocation = new Location();
         dataLocation.setPath("/data/foo/bar");


Mime
View raw message