falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkat...@apache.org
Subject [2/2] git commit: FALCON-280 Validate the ACL in Feed entity with the user submitting the entity. Contributed by Jean-Baptiste Onofré
Date Mon, 16 Jun 2014 10:58:33 GMT
FALCON-280 Validate the ACL in Feed entity with the user submitting the entity. Contributed
by Jean-Baptiste Onofré


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/b68946a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/b68946a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/b68946a0

Branch: refs/heads/master
Commit: b68946a02d6eff9314e242f73d7e97855c01957d
Parents: b501baa
Author: Venkatesh Seetharam <venkatesh@apache.org>
Authored: Mon Jun 16 16:28:17 2014 +0530
Committer: Venkatesh Seetharam <venkatesh@apache.org>
Committed: Mon Jun 16 16:28:17 2014 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 ++
 .../apache/falcon/entity/CatalogStorage.java    |  6 +++
 .../apache/falcon/entity/FileSystemStorage.java | 47 ++++++++++++++++++++
 .../java/org/apache/falcon/entity/Storage.java  | 10 +++++
 .../falcon/entity/parser/FeedEntityParser.java  | 17 +++++++
 .../falcon/entity/FileSystemStorageTest.java    | 24 ++++++++++
 .../resources/config/cluster/cluster-0.1.xml    |  2 +-
 .../workflow/engine/OozieWorkflowEngine.java    |  4 +-
 8 files changed, 110 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e6ca795..0b007e2 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,9 @@ Trunk (Unreleased)
   NEW FEATURES
 
   IMPROVEMENTS
+   FALCON-280 Validate the ACL in Feed entity with the user submitting the entity
+   (Jean-Baptiste Onofré via Venkatesh Seetharam)
+
    FALCON-445 Propagate hive credentials defined in cluster entity to
    hive-site.xml (Sowmya Ramesh via Venkatesh Seetharam)
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java b/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
index 37a05cb..f7b592d 100644
--- a/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
@@ -338,6 +338,12 @@ public class CatalogStorage implements Storage {
     }
 
     @Override
+    public void validateACL(String owner, String group, String permissions) throws FalconException
{
+        // This is not supported in Hive today as authorization is not enforced on table
and
+        // partition listing
+    }
+
+    @Override
     public String toString() {
         return "CatalogStorage{"
                 + "catalogUrl='" + catalogUrl + '\''

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
index c2b86b6..9b76d3b 100644
--- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
@@ -23,8 +23,14 @@ import org.apache.falcon.entity.v0.feed.Feed;
 import org.apache.falcon.entity.v0.feed.Location;
 import org.apache.falcon.entity.v0.feed.LocationType;
 import org.apache.falcon.entity.v0.feed.Locations;
+import org.apache.falcon.hadoop.HadoopClientFactory;
 import org.apache.falcon.security.CurrentUser;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -36,6 +42,8 @@ import java.util.List;
  */
 public class FileSystemStorage implements Storage {
 
+    private static final Logger LOG = LoggerFactory.getLogger(FileSystemStorage.class);
+
     public static final String FEED_PATH_SEP = "#";
     public static final String LOCATION_TYPE_SEP = "=";
 
@@ -227,6 +235,45 @@ public class FileSystemStorage implements Storage {
     }
 
     @Override
+    public void validateACL(String owner, String group, String permissions) throws FalconException
{
+        try {
+            FileSystem fileSystem = HadoopClientFactory.get().createProxiedFileSystem(getConf());
+            for (Location location : getLocations()) {
+                String pathString = getRelativePath(location);
+                Path path = new Path(pathString);
+                if (fileSystem.exists(path)) {
+                    FileStatus fileStatus = fileSystem.getFileStatus(path);
+                    if (!fileStatus.getOwner().equals(owner)) {
+                        LOG.error("Feed ACL owner {} doesn't match the actual file owner
{}",
+                                owner, fileStatus.getOwner());
+                        throw new FalconException("Feed ACL owner " + owner + " doesn't match
the actual file owner "
+                                + fileStatus.getOwner());
+                    }
+                }
+            }
+        } catch (Exception e) {
+            LOG.error("Can't validate ACL on storage {}", getStorageUrl(), e);
+            throw new FalconException("Can't validate storage ACL (URI " + getStorageUrl()
+ ")", e);
+        }
+    }
+
+    private Configuration getConf() {
+        Configuration conf = new Configuration();
+        conf.set(HadoopClientFactory.FS_DEFAULT_NAME_KEY, storageUrl);
+        return conf;
+    }
+
+    private String getRelativePath(Location location) {
+        // if the path contains variables, locate on the "parent" path (just before first
variable usage)
+        int index = location.getPath().indexOf(DOLLAR_EXPR_START_REGEX);
+        String pathString = location.getPath();
+        if (index != -1) {
+            pathString = pathString.substring(index);
+        }
+        return pathString;
+    }
+
+    @Override
     public String toString() {
         return "FileSystemStorage{"
                 + "storageUrl='" + storageUrl + '\''

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/common/src/main/java/org/apache/falcon/entity/Storage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/Storage.java b/common/src/main/java/org/apache/falcon/entity/Storage.java
index 60c87c5..b49410e 100644
--- a/common/src/main/java/org/apache/falcon/entity/Storage.java
+++ b/common/src/main/java/org/apache/falcon/entity/Storage.java
@@ -72,4 +72,14 @@ public interface Storage {
      * @throws FalconException an exception
      */
     boolean isIdentical(Storage toCompareAgainst) throws FalconException;
+
+    /**
+     * Check the permission on the storage, regarding owner/group/permission coming from
ACL.
+     *
+     * @param owner the owner defined in the ACL.
+     * @param group the group defined in the ACL.
+     * @param permissions the permissions defined in the ACL.
+     * @throws FalconException if the permissions are not valid.
+     */
+    void validateACL(String owner, String group, String permissions) throws FalconException;
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
index 7f87bdc..d5a8a76 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
@@ -314,6 +314,9 @@ public class FeedEntityParser extends EntityParser<Feed> {
      * Does not matter for FileSystem storage.
      */
     private void validateFeedStorage(Feed feed) throws FalconException {
+        validateUser(feed);
+        validateACL(feed);
+
         final Storage.TYPE baseFeedStorageType = FeedHelper.getStorageType(feed);
         validateMultipleSourcesExist(feed, baseFeedStorageType);
         validateUniformStorageType(feed, baseFeedStorageType);
@@ -405,4 +408,18 @@ public class FeedEntityParser extends EntityParser<Feed> {
             throw new ValidationException(buffer.toString());
         }
     }
+
+    private void validateACL(Feed feed) throws FalconException {
+        for (Cluster cluster : feed.getClusters().getClusters()) {
+            org.apache.falcon.entity.v0.cluster.Cluster clusterEntity =
+                    EntityUtil.getEntity(EntityType.CLUSTER, cluster.getName());
+            if (!EntityUtil.responsibleFor(clusterEntity.getColo())) {
+                continue;
+            }
+
+            final Storage storage = FeedHelper.createStorage(cluster, feed);
+            storage.validateACL(feed.getACL().getOwner(), feed.getACL().getGroup(),
+                    feed.getACL().getPermission());
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java b/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
index e095f8e..33e74fa 100644
--- a/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
@@ -148,6 +148,30 @@ public class FileSystemStorageTest {
         Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), absoluteUrl);
     }
 
+    @Test
+    public void testValidateACL() throws Exception {
+        final Location location = new Location();
+        location.setPath("/foo/bar");
+        location.setType(LocationType.DATA);
+        List<Location> locations = new ArrayList<Location>();
+        locations.add(location);
+
+        FileSystemStorage storage = new FileSystemStorage("jail://global:00", locations);
+        storage.validateACL(USER, USER, "rrr");
+    }
+
+    @Test
+    public void testValidateACLWithTimeVariables() throws Exception {
+        final Location location = new Location();
+        location.setPath("/foo/bar/${YEAR}/${MONTH}/${DAY}");
+        location.setType(LocationType.DATA);
+        List<Location> locations = new ArrayList<Location>();
+        locations.add(location);
+
+        FileSystemStorage storage = new FileSystemStorage("jail://global:00", locations);
+        storage.validateACL(USER, USER, "rrr");
+    }
+
     @DataProvider(name = "locationTestWithRelativePathDataProvider")
     private Object[][] createLocationTestDataWithRelativePath() {
         return new Object[][] {

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/common/src/test/resources/config/cluster/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/common/src/test/resources/config/cluster/cluster-0.1.xml b/common/src/test/resources/config/cluster/cluster-0.1.xml
index 658711d..0f5fb47 100644
--- a/common/src/test/resources/config/cluster/cluster-0.1.xml
+++ b/common/src/test/resources/config/cluster/cluster-0.1.xml
@@ -22,7 +22,7 @@
     <interfaces>
         <interface type="readonly" endpoint="hftp://localhost:50010"
                    version="0.20.2"/>
-        <interface type="write" endpoint="hdfs://localhost:8020"
+        <interface type="write" endpoint="jail://testCluster:00"
                    version="0.20.2"/>
         <interface type="execute" endpoint="localhost:8021" version="0.20.2"/>
         <interface type="workflow" endpoint="http://localhost:11000/oozie/"

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
index 096be40..7166b13 100644
--- a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
+++ b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
@@ -1123,8 +1123,8 @@ public class OozieWorkflowEngine extends AbstractWorkflowEngine {
             updateCoords(cluster, oldBundle, EntityUtil.getParallel(oldEntity), effectiveTime);
         }
 
-        if (oldBundleStatus != Job.Status.SUSPENDED &&
-                oldBundleStatus != Job.Status.PREPSUSPENDED) {
+        if (oldBundleStatus != Job.Status.SUSPENDED
+                && oldBundleStatus != Job.Status.PREPSUSPENDED) {
             //resume coords
             resumeCoords(cluster, oldBundle);
         }


Mime
View raw message