falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkat...@apache.org
Subject [6/8] incubator-falcon git commit: FALCON-889 Windows azure replication fails with "wasb" as the scheme to an HDFS file system. Contributed by Chris Nauroth
Date Fri, 14 Nov 2014 02:54:51 GMT
FALCON-889 Windows azure replication fails with "wasb" as the scheme to an HDFS file system.
Contributed by Chris Nauroth


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/60d33b61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/60d33b61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/60d33b61

Branch: refs/heads/master
Commit: 60d33b615d90cdaea2a869c24b0157e9f5a798d4
Parents: fc3094a
Author: Venkatesh Seetharam <venkatesh@apache.org>
Authored: Thu Nov 13 18:19:11 2014 -0800
Committer: Venkatesh Seetharam <venkatesh@apache.org>
Committed: Thu Nov 13 18:19:11 2014 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 ++
 .../apache/falcon/entity/FileSystemStorage.java |  3 +-
 .../entity/parser/FeedEntityParserTest.java     | 50 +++++++++++++++++++-
 3 files changed, 54 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/60d33b61/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 443eb65..f20ef0d 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -144,6 +144,9 @@ Trunk (Unreleased)
   OPTIMIZATIONS
 
   BUG FIXES
+   FALCON-889 Windows azure replication fails with "wasb" as the scheme to an
+   HDFS file system (Chris Nauroth via Venkatesh Seetharam)
+
    FALCON-885 RequestID is coming as null for all instance API calls
    (Venkatesh Seetharam)
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/60d33b61/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
index f7bd321..bbe274b 100644
--- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
@@ -273,10 +273,11 @@ public class FileSystemStorage extends Configured implements Storage
{
     @Override
     public void validateACL(AccessControlList acl) throws FalconException {
         try {
-            FileSystem fileSystem = HadoopClientFactory.get().createProxiedFileSystem(getConf());
             for (Location location : getLocations()) {
                 String pathString = getRelativePath(location);
                 Path path = new Path(pathString);
+                FileSystem fileSystem =
+                    HadoopClientFactory.get().createProxiedFileSystem(path.toUri(), getConf());
                 if (fileSystem.exists(path)) {
                     FileStatus fileStatus = fileSystem.getFileStatus(path);
                     Set<String> groups = CurrentUser.getGroupNames();

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/60d33b61/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
index 8c67a7d..8e773cb 100644
--- a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
@@ -30,7 +30,15 @@ import org.apache.falcon.entity.v0.Frequency;
 import org.apache.falcon.entity.v0.SchemaHelper;
 import org.apache.falcon.entity.v0.cluster.Cluster;
 import org.apache.falcon.entity.v0.cluster.Interfacetype;
-import org.apache.falcon.entity.v0.feed.*;
+import org.apache.falcon.entity.v0.feed.ActionType;
+import org.apache.falcon.entity.v0.feed.ClusterType;
+import org.apache.falcon.entity.v0.feed.Feed;
+import org.apache.falcon.entity.v0.feed.Location;
+import org.apache.falcon.entity.v0.feed.LocationType;
+import org.apache.falcon.entity.v0.feed.Locations;
+import org.apache.falcon.entity.v0.feed.Partition;
+import org.apache.falcon.entity.v0.feed.Partitions;
+import org.apache.falcon.entity.v0.feed.Validity;
 import org.apache.falcon.group.FeedGroupMapTest;
 import org.apache.falcon.security.CurrentUser;
 import org.apache.falcon.util.StartupProperties;
@@ -874,4 +882,44 @@ public class FeedEntityParserTest extends AbstractTestBase {
             }
         }
     }
+
+    @Test
+    public void testValidateACLForArchiveReplication() throws Exception {
+        StartupProperties.get().setProperty("falcon.security.authorization.enabled", "true");
+        Assert.assertTrue(Boolean.valueOf(
+            StartupProperties.get().getProperty("falcon.security.authorization.enabled")));
+
+        CurrentUser.authenticate(USER);
+        try {
+            InputStream stream = this.getClass().getResourceAsStream(FEED_XML);
+
+            // need a new parser since it caches authorization enabled flag
+            FeedEntityParser feedEntityParser =
+                (FeedEntityParser) EntityParserFactory.getParser(EntityType.FEED);
+            Feed feed = feedEntityParser.parse(stream);
+
+            org.apache.falcon.entity.v0.feed.Cluster feedCluster =
+                FeedHelper.getCluster(feed, "backupCluster");
+            Location location = new Location();
+            location.setType(LocationType.DATA);
+            location.setPath(
+                "s3://falcontesting@hwxasvtesting.blob.core.windows.net/{YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}");
+            Locations locations = new Locations();
+            locations.getLocations().add(location);
+            feedCluster.setLocations(locations);
+
+            Assert.assertNotNull(feed);
+            Assert.assertNotNull(feed.getACL());
+            feed.getACL().setOwner(USER);
+            feed.getACL().setGroup(getPrimaryGroupName());
+
+            try {
+                feedEntityParser.validate(feed);
+            } catch (IllegalArgumentException e) {
+                // this is normal since AWS Secret Access Key is not specified as the password
of a s3 URL
+            }
+        } finally {
+            StartupProperties.get().setProperty("falcon.security.authorization.enabled",
"false");
+        }
+    }
 }


Mime
View raw message