falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pall...@apache.org
Subject falcon git commit: FALCON-2184 Support to delete a non trusted recipe metadata from server
Date Fri, 18 Nov 2016 08:41:56 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 2c9295a9d -> b11e0570f


FALCON-2184 Support to delete a non trusted recipe metadata from server

Author: Praveen Adlakha <adlakha.praveen@gmail.com>

Reviewers: @pallavi-rao

Closes #300 from PraveenAdlakha/755 and squashes the following commits:

74e7383 [Praveen Adlakha] checkstyle issue fixed
acf2b5b [Praveen Adlakha] exception handling changed
fb3028d [Praveen Adlakha] comment's addressed
6269ba8 [Praveen Adlakha] checkstyle issues fixed
cd79f5d [Praveen Adlakha] FALCON-2181 Support for storing metadata of non trusted recipe
c483e4f [Praveen Adlakha] comments addressed
a158993 [Praveen Adlakha] test cases added and comments addressed
8fbb608 [Praveen Adlakha] WIP
63a0cce [Praveen Adlakha] sandeep's comment addressed
9ff91e0 [Praveen Adlakha] FALCON-2181 Support for storing metadata of non trusted recipe


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/b11e0570
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/b11e0570
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/b11e0570

Branch: refs/heads/master
Commit: b11e0570f74c7355b0e4ef648f80c1e6ca9a3523
Parents: 2c9295a
Author: Praveen Adlakha <adlakha.praveen@gmail.com>
Authored: Fri Nov 18 14:11:35 2016 +0530
Committer: Pallavi Rao <pallavi.rao@inmobi.com>
Committed: Fri Nov 18 14:11:35 2016 +0530

----------------------------------------------------------------------
 .../java/org/apache/falcon/cli/FalconCLI.java   |  2 +-
 .../apache/falcon/cli/FalconExtensionCLI.java   |  6 +++++
 .../falcon/client/FalconCLIConstants.java       |  1 +
 .../org/apache/falcon/client/FalconClient.java  | 10 ++++++++-
 .../persistence/ExtensionMetadataBean.java      |  4 +++-
 .../persistence/PersistenceConstants.java       |  2 ++
 .../apache/falcon/extensions/ExtensionType.java |  2 +-
 .../extensions/jdbc/ExtensionMetaStore.java     | 23 ++++++++++++++++++++
 .../falcon/extensions/store/ExtensionStore.java | 15 +++++++++++++
 .../extensions/jdbc/ExtensionMetaStoreTest.java |  2 +-
 .../store/AbstractTestExtensionStore.java       |  1 -
 .../resource/extensions/ExtensionManager.java   | 16 +++++++++++++-
 12 files changed, 77 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java
----------------------------------------------------------------------
diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java
index 0dd11f6..5b5f9ff 100644
--- a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java
+++ b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java
@@ -100,7 +100,7 @@ public class FalconCLI {
                 metadataCLI.createMetadataOptions(), true);
         parser.addCommand(FalconCLIConstants.EXTENSION_CMD, "",
                 "Extension operations like enumerate, definition, describe, list, instances,
"
-                        + "submit, submitAndSchedule, schedule, suspend, resume, delete,
update, validate",
+                        + "submit, submitAndSchedule, schedule, suspend, resume, delete,
update, validate,unregister",
                 extensionCLI.createExtensionOptions(), true);
         parser.addCommand(FalconCLIConstants.VERSION_OPT, "", "show client version", new
Options(), false);
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
----------------------------------------------------------------------
diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
index dafd7a8..1bb27a0 100644
--- a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
+++ b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
@@ -49,6 +49,7 @@ public class FalconExtensionCLI {
     public static final String DEFINITION_OPT = "definition";
     public static final String DESCRIBE_OPT = "describe";
     public static final String INSTANCES_OPT = "instances";
+    public static final String UNREGISTER_OPT = "unregister";
 
     // Input parameters
     public static final String ENTENSION_NAME_OPT = "extensionName";
@@ -79,6 +80,9 @@ public class FalconExtensionCLI {
         } else if (optionsList.contains(DESCRIBE_OPT)) {
             validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
             result = client.getExtensionDescription(extensionName);
+        } else if (optionsList.contains(UNREGISTER_OPT)) {
+            validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
+            result = client.unregisterExtension(extensionName);
         } else if (optionsList.contains(FalconCLIConstants.SUBMIT_OPT)) {
             validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
             validateRequiredParameter(filePath, FalconCLIConstants.FILE_PATH_OPT);
@@ -153,6 +157,7 @@ public class FalconExtensionCLI {
         Option suspend = new Option(FalconCLIConstants.SUSPEND_OPT, false, "Suspend an extension
job");
         Option resume = new Option(FalconCLIConstants.RESUME_OPT, false, "Resume an extension
job");
         Option delete = new Option(FalconCLIConstants.DELETE_OPT, false, "Delete an extension
job");
+        Option unregister = new Option(FalconCLIConstants.UREGISTER, false, "Delete metadata
of extension job");
 
         OptionGroup group = new OptionGroup();
         group.addOption(enumerate);
@@ -168,6 +173,7 @@ public class FalconExtensionCLI {
         group.addOption(suspend);
         group.addOption(resume);
         group.addOption(delete);
+        group.addOption(unregister);
         extensionOptions.addOptionGroup(group);
 
         Option url = new Option(FalconCLIConstants.URL_OPTION, true, "Falcon URL");

http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java b/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java
index 04f1599..8b15f86 100644
--- a/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java
+++ b/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java
@@ -217,4 +217,5 @@ public final class FalconCLIConstants {
     public static final String DIRECTION_OPT_DESCRIPTION = "edge direction property";
     public static final String DEBUG_OPTION_DESCRIPTION = "Use debug mode to see debugging
statements on stdout";
     public static final String DO_AS_DESCRIPTION = "doAs user";
+    public static final String UREGISTER = "unregister";
 }

http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/client/src/main/java/org/apache/falcon/client/FalconClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java
index 8f77fad..ef9df7f 100644
--- a/client/src/main/java/org/apache/falcon/client/FalconClient.java
+++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java
@@ -346,7 +346,8 @@ public class FalconClient extends AbstractFalconClient {
         SCHEDULE("api/extension/schedule", HttpMethod.POST, MediaType.TEXT_XML),
         SUSPEND("api/extension/suspend", HttpMethod.POST, MediaType.TEXT_XML),
         RESUME("api/extension/resume", HttpMethod.POST, MediaType.TEXT_XML),
-        DELETE("api/extension/delete", HttpMethod.POST, MediaType.TEXT_XML);
+        DELETE("api/extension/delete", HttpMethod.POST, MediaType.TEXT_XML),
+        UNREGISTER("api/extension/unregister/", HttpMethod.POST, MediaType.TEXT_PLAIN);
 
         private String path;
         private String method;
@@ -1021,6 +1022,13 @@ public class FalconClient extends AbstractFalconClient {
         return getResponse(String.class, clientResponse);
     }
 
+    public String unregisterExtension(final String extensionName) {
+        ClientResponse clientResponse = new ResourceBuilder()
+                .path(ExtensionOperations.UNREGISTER.path, extensionName)
+                .call(ExtensionOperations.UNREGISTER);
+        return getResponse(String.class, clientResponse);
+    }
+
     public String getExtensionDefinition(final String extensionName)  {
         ClientResponse clientResponse = new ResourceBuilder()
                 .path(ExtensionOperations.DEFINITION.path, extensionName)

http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java
b/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java
index ac1ded6..f4208ad 100644
--- a/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java
+++ b/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java
@@ -38,7 +38,9 @@ import java.util.Date;
 @NamedQueries({
         @NamedQuery(name = PersistenceConstants.GET_ALL_EXTENSIONS, query = "select OBJECT(a)
from ExtensionMetadataBean a "),
         @NamedQuery(name = PersistenceConstants.GET_EXTENSION_LOCATION, query = "select a.location
from ExtensionMetadataBean a where a.extensionName = :extensionName"),
-        @NamedQuery(name = PersistenceConstants.DELETE_EXTENSIONS_OF_TYPE, query = "delete
from ExtensionMetadataBean a where a.extensionType = :extensionType ")
+        @NamedQuery(name = PersistenceConstants.DELETE_EXTENSIONS_OF_TYPE, query = "delete
from ExtensionMetadataBean a where a.extensionType = :extensionType "),
+        @NamedQuery(name = PersistenceConstants.DELETE_EXTENSION, query = "delete from ExtensionMetadataBean
a where a.extensionName = :extensionName "),
+        @NamedQuery(name = PersistenceConstants.GET_EXTENSION, query = "select OBJECT(a)
from ExtensionMetadataBean a where a.extensionName = :extensionName")
 })
 //RESUME CHECKSTYLE CHECK  LineLengthCheck
 public class ExtensionMetadataBean {

http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java
b/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java
index dbbf06d..d0e8e9c 100644
--- a/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java
+++ b/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java
@@ -70,4 +70,6 @@ public final class PersistenceConstants {
     public static final String GET_ALL_EXTENSIONS = "GET_ALL_EXTENSIONS";
     public static final String GET_EXTENSION_LOCATION = "GET_EXTENSION_LOCATION";
     public static final String DELETE_EXTENSIONS_OF_TYPE = "DELETE_EXTENSIONS_OF_TYPE";
+    public static final String DELETE_EXTENSION = "DELETE_EXTENSION";
+    public static final String GET_EXTENSION = "GET_EXTENSION";
 }

http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java
index c7e45cc..c4621cc 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java
@@ -34,4 +34,4 @@ public enum ExtensionType {
     public String toString(){
         return text;
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
index b2f7e7d..2c6003d 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
@@ -53,6 +53,17 @@ public class ExtensionMetaStore {
         }
     }
 
+    public Boolean checkIfExtensionExists(String extensionName){
+        EntityManager entityManager = getEntityManager();
+        beginTransaction(entityManager);
+        Query q = entityManager.createNamedQuery(PersistenceConstants.GET_EXTENSION);
+        q.setParameter("extensionName", extensionName);
+        if (q.getResultList().size() > 0){
+            return true;
+        }
+        return false;
+    }
+
     public List<ExtensionMetadataBean> getAllExtensions(){
         EntityManager entityManager = getEntityManager();
         beginTransaction(entityManager);
@@ -88,6 +99,18 @@ public class ExtensionMetaStore {
         }
     }
 
+    public void deleteExtensionMetadata(String extensionName){
+        EntityManager entityManager = getEntityManager();
+        beginTransaction(entityManager);
+        Query q = entityManager.createNamedQuery(PersistenceConstants.DELETE_EXTENSION);
+        q.setParameter("extensionName", extensionName);
+        try{
+            q.executeUpdate();
+        } finally {
+            commitAndCloseTransaction(entityManager);
+        }
+    }
+
     private void beginTransaction(EntityManager entityManager) {
         entityManager.getTransaction().begin();
     }

http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
index 61804f9..ffb4341 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
@@ -24,6 +24,7 @@ import org.apache.falcon.extensions.AbstractExtension;
 import org.apache.falcon.extensions.ExtensionType;
 import org.apache.falcon.extensions.jdbc.ExtensionMetaStore;
 import org.apache.falcon.hadoop.HadoopClientFactory;
+import org.apache.falcon.entity.parser.ValidationException;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IOUtils;
@@ -250,6 +251,20 @@ public final class ExtensionStore {
         return extesnionList;
     }
 
+    public String deleteExtensionMetadata(final String extensionName) throws ValidationException{
+        ExtensionType extensionType = AbstractExtension.isExtensionTrusted(extensionName)
? ExtensionType.TRUSTED
+                : ExtensionType.CUSTOM;
+        if (extensionType.equals(ExtensionType.TRUSTED)){
+            throw new ValidationException(extensionName + " is trusted cannot be deleted.");
+        }
+        if (metaStore.checkIfExtensionExists(extensionName)) {
+            metaStore.deleteExtensionMetadata(extensionName);
+            return "Deleted extension:" + extensionName;
+        }else {
+            return "Extension:" + extensionName + " is not registered with Falcon.";
+        }
+    }
+
     public String getResource(final String extensionName, final String resourceName) throws
StoreAccessException {
         Map<String, String> resources = getExtensionArtifacts(extensionName);
         if (resources.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
----------------------------------------------------------------------
diff --git a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
index 61b26d6..7e0db0d 100644
--- a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
+++ b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
@@ -1,4 +1,3 @@
-
 /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -60,6 +59,7 @@ public class ExtensionMetaStoreTest extends AbstractTestExtensionStore {
     public void dbOpertaions(){
         //insert
         stateStore.storeExtensionMetadataBean("test1", "test_location", ExtensionType.TRUSTED,
"test_description");
+
         Assert.assertEquals(stateStore.getAllExtensions().size(), 1);
         //check data
         Assert.assertEquals(stateStore.getLocation("test1"), "test_location");

http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java
b/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java
index cd94eca..97fe287 100644
--- a/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java
+++ b/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java
@@ -83,7 +83,6 @@ public class AbstractTestExtensionStore {
         falconJPAService.init();
         new ExtensionService().init();
         store = ExtensionService.getExtensionStore();
-        FalconJPAService falconJPAService = FalconJPAService.get();
         fileSystem = HadoopClientFactory.get().createFalconFileSystem(new Configuration(true));
         extensionStorePath = new URI(StartupProperties.get().getProperty(ExtensionStore.EXTENSION_STORE_URI)).getPath();
         extensionStoreSetup(resourceClass);

http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
index 138109c..5669b8f 100644
--- a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
+++ b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
@@ -420,6 +420,21 @@ public class ExtensionManager extends AbstractSchedulableEntityManager
{
         }
     }
 
+    @POST
+    @Path("unregister/{extension-name}")
+    @Consumes({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
+    @Produces(MediaType.TEXT_PLAIN)
+    public String deleteExtensionMetadata(
+            @PathParam("extension-name") String extensionName){
+        checkIfExtensionServiceIsEnabled();
+        validateExtensionName(extensionName);
+        try {
+            return ExtensionStore.get().deleteExtensionMetadata(extensionName);
+        } catch (Throwable e) {
+            throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR);
+        }
+    }
+
     @GET
     @Path("definition/{extension-name}")
     @Produces({MediaType.APPLICATION_JSON})
@@ -463,7 +478,6 @@ public class ExtensionManager extends AbstractSchedulableEntityManager
{
         return results;
     }
 
-
     private List<Entity> generateEntities(String extensionName, HttpServletRequest
request)
         throws FalconException, IOException {
         // get entities for extension job


Mime
View raw message