falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From suh...@apache.org
Subject falcon git commit: FALCON-1020 validate command produces different results when run via prism and server. Contributed by pavan kumar kolamuri
Date Mon, 09 Mar 2015 15:14:32 GMT
Repository: falcon
Updated Branches:
  refs/heads/master c790761e0 -> 0c6f3911d


FALCON-1020 validate command produces different results when run via prism and server. Contributed
by pavan kumar kolamuri


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/0c6f3911
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/0c6f3911
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/0c6f3911

Branch: refs/heads/master
Commit: 0c6f3911d2972f266f7e0aca003560b5d530ca7a
Parents: c790761
Author: Suhas Vasu <suhas.v@inmobi.com>
Authored: Mon Mar 9 20:44:10 2015 +0530
Committer: Suhas Vasu <suhas.v@inmobi.com>
Committed: Mon Mar 9 20:44:10 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |   6 +
 FALCON-1063.patch                               | 204 -------------------
 client/src/main/resources/feed-0.1.xsd          |   7 +-
 .../apache/falcon/entity/FileSystemStorage.java |   2 +-
 .../proxy/SchedulableEntityManagerProxy.java    |  23 ++-
 5 files changed, 34 insertions(+), 208 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 453e44c..3b9b243 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -81,6 +81,9 @@ Trunk (Unreleased)
    Seetharam)
 
   OPTIMIZATIONS
+   FALCON-1063 Falcon CLI list entities operation throws NullPointerException
+   (Pallavi Rao via Suhas Vasu)
+
    FALCON-987 params command inconsistent behavior
    (pavan kumar kolamuri via Suhas Vasu)
 
@@ -98,6 +101,9 @@ Trunk (Unreleased)
    (Suhas vasu)
 
   BUG FIXES
+   FALCON-1020 validate command produces different results when run
+   via prism and server (pavan kumar kolamuri via Suhas Vasu)
+
    FALCON-950 Rerun does not work on succeeded instances (Suhas Vasu)
 
    FALCON-1048 Incorrect documentation for feed instacnce listing api. (Suhas

http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/FALCON-1063.patch
----------------------------------------------------------------------
diff --git a/FALCON-1063.patch b/FALCON-1063.patch
deleted file mode 100644
index 5aea017..0000000
--- a/FALCON-1063.patch
+++ /dev/null
@@ -1,204 +0,0 @@
-diff --git a/common/src/main/java/org/apache/falcon/cleanup/AbstractCleanupHandler.java b/common/src/main/java/org/apache/falcon/cleanup/AbstractCleanupHandler.java
-index 5a4dce6..85d7263 100644
---- a/common/src/main/java/org/apache/falcon/cleanup/AbstractCleanupHandler.java
-+++ b/common/src/main/java/org/apache/falcon/cleanup/AbstractCleanupHandler.java
-@@ -116,11 +116,11 @@ public abstract class AbstractCleanupHandler {
-                                                   Entity entity) throws FalconException
{
-         try {
-             final AccessControlList acl = entity.getACL();
--            if (acl == null) {
--                throw new FalconException("ACL for entity " + entity.getName() + " is empty");
-+            // To support backward compatibility, will only use the ACL owner only if present
-+            if (acl != null) {
-+                CurrentUser.authenticate(acl.getOwner()); // proxy user
-             }
- 
--            CurrentUser.authenticate(acl.getOwner()); // proxy user
-             return HadoopClientFactory.get().createProxiedFileSystem(
-                     ClusterHelper.getConfiguration(cluster));
-         } catch (Exception e) {
-diff --git a/common/src/main/java/org/apache/falcon/security/SecurityUtil.java b/common/src/main/java/org/apache/falcon/security/SecurityUtil.java
-index b9fd37e..861f80f 100644
---- a/common/src/main/java/org/apache/falcon/security/SecurityUtil.java
-+++ b/common/src/main/java/org/apache/falcon/security/SecurityUtil.java
-@@ -19,11 +19,13 @@
- package org.apache.falcon.security;
- 
- import org.apache.falcon.FalconException;
-+import org.apache.falcon.entity.v0.Entity;
- import org.apache.falcon.util.ReflectionUtils;
- import org.apache.falcon.util.StartupProperties;
- import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
- import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
- 
-+import java.io.IOException;
- import java.net.InetAddress;
- import java.net.UnknownHostException;
- 
-@@ -104,4 +106,16 @@ public final class SecurityUtil {
-                 "org.apache.falcon.security.DefaultAuthorizationProvider");
-         return ReflectionUtils.getInstanceByClassName(providerClassName);
-     }
-+
-+    public static void tryProxy(Entity entity) throws IOException, FalconException {
-+        if (entity != null && entity.getACL() != null && SecurityUtil.isAuthorizationEnabled())
{
-+            final String aclOwner = entity.getACL().getOwner();
-+            final String aclGroup = entity.getACL().getGroup();
-+
-+            if (SecurityUtil.getAuthorizationProvider().shouldProxy(
-+                    CurrentUser.getAuthenticatedUGI(), aclOwner, aclGroup)) {
-+                CurrentUser.proxy(aclOwner, aclGroup);
-+            }
-+        }
-+    }
- }
-diff --git a/common/src/test/java/org/apache/falcon/cleanup/LogCleanupServiceTest.java b/common/src/test/java/org/apache/falcon/cleanup/LogCleanupServiceTest.java
-index 8e2e544..0df59b2 100644
---- a/common/src/test/java/org/apache/falcon/cleanup/LogCleanupServiceTest.java
-+++ b/common/src/test/java/org/apache/falcon/cleanup/LogCleanupServiceTest.java
-@@ -55,6 +55,8 @@ public class LogCleanupServiceTest extends AbstractTestBase {
-         + "sample2" + "/logs/job-2010-01-01-01-00/000");
-     private final Path instanceLogPath4 = new Path("/projects/falcon/staging/falcon/workflows/process/"
-         + "sample" + "/logs/latedata/2010-01-01-01-00");
-+    private final Path instanceLogPath5 = new Path("/projects/falcon/staging/falcon/workflows/process/"
-+            + "sample3" + "/logs/job-2010-01-01-01-00/000");
-     private final Path feedInstanceLogPath = new Path("/projects/falcon/staging/falcon/workflows/feed/"
-         + "impressionFeed" + "/logs/job-2010-01-01-01-00/testCluster/000");
-     private final Path feedInstanceLogPath1 = new Path("/projects/falcon/staging/falcon/workflows/feed/"
-@@ -90,15 +92,22 @@ public class LogCleanupServiceTest extends AbstractTestBase {
-         Process otherProcess = (Process) process.copy();
-         otherProcess.setName("sample2");
-         otherProcess.setFrequency(new Frequency("days(1)"));
-+        Process noACLProcess = (Process) process.copy();
-+        noACLProcess.setName("sample3");
-+        noACLProcess.setACL(null);
-         ConfigurationStore.get().remove(EntityType.PROCESS,
-                 otherProcess.getName());
-         ConfigurationStore.get().publish(EntityType.PROCESS, otherProcess);
-+        ConfigurationStore.get().remove(EntityType.PROCESS,
-+                noACLProcess.getName());
-+        ConfigurationStore.get().publish(EntityType.PROCESS, noACLProcess);
- 
-         fs.mkdirs(instanceLogPath);
-         fs.mkdirs(instanceLogPath1);
-         fs.mkdirs(instanceLogPath2);
-         fs.mkdirs(instanceLogPath3);
-         fs.mkdirs(instanceLogPath4);
-+        fs.mkdirs(instanceLogPath5);
- 
-         // fs.setTimes wont work on dirs
-         fs.createNewFile(new Path(instanceLogPath, "oozie.log"));
-@@ -138,6 +147,7 @@ public class LogCleanupServiceTest extends AbstractTestBase {
-         Assert.assertFalse(fs.exists(instanceLogPath));
-         Assert.assertFalse(fs.exists(instanceLogPath1));
-         Assert.assertFalse(fs.exists(instanceLogPath2));
-+        Assert.assertFalse(fs.exists(instanceLogPath5));
-         Assert.assertTrue(fs.exists(instanceLogPath3));
-     }
- 
-diff --git a/common/src/test/java/org/apache/falcon/security/SecurityUtilTest.java b/common/src/test/java/org/apache/falcon/security/SecurityUtilTest.java
-index a36d916..7f9b405 100644
---- a/common/src/test/java/org/apache/falcon/security/SecurityUtilTest.java
-+++ b/common/src/test/java/org/apache/falcon/security/SecurityUtilTest.java
-@@ -18,10 +18,17 @@
- 
- package org.apache.falcon.security;
- 
-+
-+import org.apache.falcon.FalconException;
-+import org.apache.falcon.entity.v0.process.*;
-+import org.apache.falcon.entity.v0.process.Process;
- import org.apache.falcon.util.StartupProperties;
-+import org.mockito.Mockito;
- import org.testng.Assert;
- import org.testng.annotations.Test;
- 
-+import java.io.IOException;
-+
- /**
-  * Unit test for Security utils.
-  */
-@@ -81,4 +88,25 @@ public class SecurityUtilTest {
-         Assert.assertEquals(SecurityUtil.getAuthorizationProvider().getClass(),
-                 DefaultAuthorizationProvider.class);
-     }
-+
-+    @Test
-+    public void testTryProxy() throws IOException, FalconException {
-+        Process process = Mockito.mock(Process.class);
-+        StartupProperties.get().setProperty("falcon.security.authorization.enabled", "true");
-+        final String currentUser = System.getProperty("user.name");
-+
-+        // When ACL not specified
-+        CurrentUser.authenticate(currentUser);
-+        SecurityUtil.tryProxy(process);
-+        Assert.assertEquals(CurrentUser.getUser(), currentUser);
-+
-+        ACL acl = new ACL();
-+        acl.setOwner("testuser");
-+        acl.setGroup("users");
-+        Mockito.when(process.getACL()).thenReturn(acl);
-+
-+        // When ACL is specified
-+        SecurityUtil.tryProxy(process);
-+        Assert.assertEquals(CurrentUser.getUser(), "testuser");
-+    }
- }
-diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
-index 9a044d9..67a66c6 100644
---- a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
-+++ b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
-@@ -406,23 +406,13 @@ public abstract class AbstractEntityManager {
-                             + "Can't be submitted again. Try removing before submitting.");
-         }
- 
--        tryProxy(entity); // proxy before validating since FS/Oozie needs to be proxied
-+        SecurityUtil.tryProxy(entity); // proxy before validating since FS/Oozie needs to
be proxied
-         validate(entity);
-         configStore.publish(entityType, entity);
-         LOG.info("Submit successful: ({}): {}", type, entity.getName());
-         return entity;
-     }
- 
--    private void tryProxy(Entity entity) throws IOException, FalconException {
--        final String aclOwner = entity.getACL().getOwner();
--        final String aclGroup = entity.getACL().getGroup();
--        if (SecurityUtil.isAuthorizationEnabled()
--                && SecurityUtil.getAuthorizationProvider().shouldProxy(
--                    CurrentUser.getAuthenticatedUGI(), aclOwner, aclGroup)) {
--            CurrentUser.proxy(aclOwner, aclGroup);
--        }
--    }
--
-     /**
-      * KLUDGE - Until ACL is mandated entity passed should be decorated for equals check
to pass.
-      * existingEntity in config store will have teh decoration and equals check fails
-@@ -646,7 +636,7 @@ public abstract class AbstractEntityManager {
-                 // the user who requested list query has no permission to access this entity.
Skip this entity
-                 continue;
-             }
--            tryProxy(entity);
-+            SecurityUtil.tryProxy(entity);
- 
-             List<String> tags = EntityUtil.getTags(entity);
-             List<String> pipelines = EntityUtil.getPipelines(entity);
-diff --git a/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java
b/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java
-index 6b022c9..15e94cd 100644
---- a/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java
-+++ b/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java
-@@ -141,14 +141,7 @@ public class FalconAuthorizationFilter implements Filter {
-         try {
-             EntityType type = EntityType.getEnum(entityType);
-             Entity entity = EntityUtil.getEntity(type, entityName);
--            if (entity != null && entity.getACL() != null) {
--                final String aclOwner = entity.getACL().getOwner();
--                final String aclGroup = entity.getACL().getGroup();
--                if (authorizationProvider.shouldProxy(
--                        authenticatedUGI, aclOwner, aclGroup)) {
--                    CurrentUser.proxy(aclOwner, aclGroup);
--                }
--            }
-+            SecurityUtil.tryProxy(entity);
-         } catch (FalconException ignore) {
-             // do nothing
-         }

http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/client/src/main/resources/feed-0.1.xsd
----------------------------------------------------------------------
diff --git a/client/src/main/resources/feed-0.1.xsd b/client/src/main/resources/feed-0.1.xsd
index 0168a35..a228b29 100644
--- a/client/src/main/resources/feed-0.1.xsd
+++ b/client/src/main/resources/feed-0.1.xsd
@@ -286,7 +286,7 @@
             </xs:documentation>
         </xs:annotation>
         <xs:attribute type="location-type" name="type" use="required"/>
-        <xs:attribute type="xs:string" name="path" use="required"/>
+        <xs:attribute type="non-empty-string" name="path" use="required"/>
     </xs:complexType>
     <xs:complexType name="partition">
         <xs:attribute type="IDENTIFIER" name="name" use="required"/>
@@ -391,4 +391,9 @@
         </xs:annotation>
         <xs:attribute type="xs:string" name="uri" use="required"/>
     </xs:complexType>
+    <xs:simpleType name="non-empty-string">
+        <xs:restriction base="xs:string">
+            <xs:minLength value="1"/>
+        </xs:restriction>
+    </xs:simpleType>
 </xs:schema>

http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
index fe93048..1ba7b9d 100644
--- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
@@ -199,7 +199,7 @@ public class FileSystemStorage extends Configured implements Storage {
             }
         }
 
-        if (locationForType == null) {
+        if (locationForType == null || StringUtils.isEmpty(locationForType.getPath())) {
             return null;
         }
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
index 5f711ee..7ba289a 100644
--- a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
+++ b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
@@ -163,8 +163,27 @@ public class SchedulableEntityManagerProxy extends AbstractSchedulableEntityMana
     @Consumes({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
     @Produces({MediaType.TEXT_XML, MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON})
     @Override
-    public APIResult validate(@Context HttpServletRequest request, @PathParam("type") String
type) {
-        return super.validate(request, type);
+    public APIResult validate(@Context final HttpServletRequest request, @PathParam("type")
final String type) {
+        final HttpServletRequest bufferedRequest = getBufferedRequest(request);
+        EntityType entityType = EntityType.getEnum(type);
+        final Entity entity;
+        try {
+            entity = deserializeEntity(bufferedRequest, entityType);
+            bufferedRequest.getInputStream().reset();
+        } catch (Exception e) {
+            throw FalconWebException.newException("Unable to parse the request", Response.Status.BAD_REQUEST);
+        }
+        return new EntityProxy(type, entity.getName()) {
+            @Override
+            protected Set<String> getColosToApply() {
+                return getApplicableColos(type, entity);
+            }
+
+            @Override
+            protected APIResult doExecute(String colo) throws FalconException {
+                return getEntityManager(colo).invoke("validate", bufferedRequest, type);
+            }
+        }.execute();
     }
 
     @DELETE


Mime
View raw message