falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pall...@apache.org
Subject falcon git commit: FALCON-2276 Falcon Trusted extensions submission and validation was failing
Date Fri, 10 Feb 2017 03:12:09 GMT
Repository: falcon
Updated Branches:
  refs/heads/0.11 2368bccf1 -> 6c105297e


FALCON-2276 Falcon Trusted extensions submission and validation was failing

Author: sandeep <sandysmdl@gmail.com>

Reviewers: @pallavi-rao

Closes #360 from sandeepSamudrala/FALCON-2276 and squashes the following commits:

84169d6 [sandeep] FALCON-2276 Falcon Trusted extensions submission and validation was failing
a3bd0e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
db425c5 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
3f67fed [sandeep] Merge branch 'master' of https://github.com/apache/falcon
cb2b00d [sandeep] Merge branch 'master' of https://github.com/apache/falcon
79e8d64 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
7de7798 [sandeep] go -b FALCON-2263Merge branch 'master' of https://github.com/apache/falcon
c5da0a2 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
7e16263 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
a234d94 [sandeep] FALCON-2231 Incoporated review comments and small fixes for duplicate submission
and colo addition to schedule command
26e3350 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
73fbf75 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
cc28658 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
089b10d [sandeep] Merge branch 'master' of https://github.com/apache/falcon
456d4ee [sandeep] Merge branch 'master' of https://github.com/apache/falcon
0cf9af6 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
4a2e23e [sandeep] Merge branch 'master' of https://github.com/apache/falcon
b1546ed [sandeep] Merge branch 'master' of https://github.com/apache/falcon
0a433fb [sandeep] Merge branch 'master' of https://github.com/apache/falcon
194f36a [sandeep] Merge branch 'master' of https://github.com/apache/falcon
e0ad358 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
f96a084 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
9cf36e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
bbca081 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
48f6afa [sandeep] Merge branch 'master' of https://github.com/apache/falcon
250cc46 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
d0393e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
a178805 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
d6dc8bf [sandeep] Merge branch 'master' of https://github.com/apache/falcon
1bb8d3c [sandeep] Merge branch 'master' of https://github.com/apache/falcon
c065566 [sandeep] reverting last line changes made
1a4dcd2 [sandeep] rebased and resolved the conflicts from master
271318b [sandeep] FALCON-2097. Adding UT to the new method for getting next instance time
with Delay.
a94d4fe [sandeep] rebasing from master
9e68a57 [sandeep] FALCON-298. Feed update with replication delay creates holes

(cherry picked from commit bf027e16995a0487296517fc0024831733e4002a)
Signed-off-by: Pallavi Rao <pallavi.rao@inmobi.com>


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/6c105297
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/6c105297
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/6c105297

Branch: refs/heads/0.11
Commit: 6c105297e05a49e09828c4caa23433b2f38d7846
Parents: 2368bcc
Author: sandeep <sandysmdl@gmail.com>
Authored: Fri Feb 10 08:41:40 2017 +0530
Committer: Pallavi Rao <pallavi.rao@inmobi.com>
Committed: Fri Feb 10 08:42:01 2017 +0530

----------------------------------------------------------------------
 .../org/apache/falcon/client/FalconClient.java   |  4 ++--
 .../org/apache/falcon/extensions/Extension.java  |  4 ++++
 .../resource/proxy/ExtensionManagerProxy.java    | 19 ++-----------------
 3 files changed, 8 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/6c105297/client/src/main/java/org/apache/falcon/client/FalconClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java
index 25eaeb5..cf02619 100644
--- a/client/src/main/java/org/apache/falcon/client/FalconClient.java
+++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java
@@ -1169,7 +1169,7 @@ public class FalconClient extends AbstractFalconClient {
     private List<Entity> getEntities(String extensionName, String jobName, InputStream
configStream,
                                      String extensionType, String extensionBuildLocation)
{
         List<Entity> entities = null;
-        if (!extensionType.equals(ExtensionType.CUSTOM.name())) {
+        if (!extensionType.equals(ExtensionType.TRUSTED.toString())) {
             try {
                 entities = ExtensionHandler.loadAndPrepare(extensionName, jobName, configStream,
                         extensionBuildLocation);
@@ -1209,7 +1209,7 @@ public class FalconClient extends AbstractFalconClient {
                                           final String configPath, final String doAsUser)
{
         String extensionType = ExtensionHandler.getExtensionType(extensionName, getExtensionDetailJson(extensionName));
         InputStream configStream = getServletInputStream(configPath);
-        if (ExtensionType.TRUSTED.name().equalsIgnoreCase(extensionType)) {
+        if (extensionType.equals(ExtensionType.TRUSTED.toString())) {
             ClientResponse clientResponse = new ResourceBuilder()
                     .path(ExtensionOperations.VALIDATE.path, extensionName)
                     .addQueryParam(DO_AS_OPT, doAsUser)

http://git-wip-us.apache.org/repos/asf/falcon/blob/6c105297/extensions/src/main/java/org/apache/falcon/extensions/Extension.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/Extension.java b/extensions/src/main/java/org/apache/falcon/extensions/Extension.java
index 3869718..6b94323 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/Extension.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/Extension.java
@@ -21,6 +21,7 @@ package org.apache.falcon.extensions;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.falcon.FalconException;
 import org.apache.falcon.Pair;
+import org.apache.falcon.entity.EntityUtil;
 import org.apache.falcon.entity.v0.Entity;
 import org.apache.falcon.entity.v0.feed.Schema;
 import org.apache.falcon.extensions.store.ExtensionStore;
@@ -109,6 +110,9 @@ public class Extension implements ExtensionBuilder {
             throw new FalconException("Entity created from the extension template cannot
be null");
         }
         LOG.info("Extension processing complete");
+        // add tags on extension name and job
+        String jobName = configProperties.getProperty(ExtensionProperties.JOB_NAME.getName());
+        EntityUtil.applyTags(extensionName, jobName, Collections.singletonList(entity));
         return Collections.singletonList(entity);
     }
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/6c105297/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java
b/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java
index 5652ca4..65618fe 100644
--- a/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java
+++ b/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java
@@ -28,7 +28,6 @@ import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Properties;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
@@ -58,7 +57,6 @@ import org.apache.falcon.entity.v0.EntityType;
 import org.apache.falcon.entity.v0.feed.Feed;
 import org.apache.falcon.entity.v0.process.Process;
 import org.apache.falcon.extensions.Extension;
-import org.apache.falcon.extensions.ExtensionProperties;
 import org.apache.falcon.extensions.ExtensionService;
 import org.apache.falcon.extensions.ExtensionType;
 import org.apache.falcon.extensions.jdbc.ExtensionMetaStore;
@@ -300,7 +298,7 @@ public class ExtensionManagerProxy extends AbstractExtensionManager {
         List<Entity> entities;
         TreeMap<EntityType, List<Entity>> entityMap = new TreeMap<>();
         if (ExtensionType.TRUSTED.equals(extensionType)) {
-            entities = generateEntities(extensionName, config);
+            entities = extension.getEntities(extensionName, config);
             List<Entity> trustedFeeds = new ArrayList<>();
             List<Entity> trustedProcesses = new ArrayList<>();
             for (Entity entity : entities) {
@@ -568,7 +566,7 @@ public class ExtensionManagerProxy extends AbstractExtensionManager {
             throw FalconWebException.newAPIException("Extension validation is supported only
for trusted extensions");
         }
         try {
-            List<Entity> entities = generateEntities(extensionName, request.getInputStream());
+            List<Entity> entities = extension.getEntities(extensionName, request.getInputStream());
             for (Entity entity : entities) {
                 super.validate(entity);
             }
@@ -719,19 +717,6 @@ public class ExtensionManagerProxy extends AbstractExtensionManager {
         }
     }
 
-    private List<Entity> generateEntities(String extensionName, InputStream configStream)
-        throws FalconException, IOException {
-        // get entities for extension job
-        Properties properties = new Properties();
-        properties.load(configStream);
-        List<Entity> entities = extension.getEntities(extensionName, configStream);
-
-        // add tags on extension name and job
-        String jobName = properties.getProperty(ExtensionProperties.JOB_NAME.getName());
-        EntityUtil.applyTags(extensionName, jobName, entities);
-        return entities;
-    }
-
     private static void checkIfExtensionServiceIsEnabled() {
         if (!Services.get().isRegistered(ExtensionService.SERVICE_NAME)) {
             LOG.error(ExtensionService.SERVICE_NAME + " is not enabled.");


Mime
View raw message