Return-Path: X-Original-To: apmail-falcon-commits-archive@minotaur.apache.org Delivered-To: apmail-falcon-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7FB0611B41 for ; Tue, 19 Aug 2014 23:37:28 +0000 (UTC) Received: (qmail 96169 invoked by uid 500); 19 Aug 2014 23:37:28 -0000 Delivered-To: apmail-falcon-commits-archive@falcon.apache.org Received: (qmail 96134 invoked by uid 500); 19 Aug 2014 23:37:28 -0000 Mailing-List: contact commits-help@falcon.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@falcon.incubator.apache.org Delivered-To: mailing list commits@falcon.incubator.apache.org Received: (qmail 96125 invoked by uid 99); 19 Aug 2014 23:37:28 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 19 Aug 2014 23:37:28 +0000 X-ASF-Spam-Status: No, hits=-2000.7 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Tue, 19 Aug 2014 23:37:25 +0000 Received: (qmail 95668 invoked by uid 99); 19 Aug 2014 23:37:05 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 19 Aug 2014 23:37:05 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id C128E8C1604; Tue, 19 Aug 2014 23:37:04 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: venkatesh@apache.org To: commits@falcon.incubator.apache.org Date: Tue, 19 Aug 2014 23:37:04 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/2] FALCON-470 Add support for pagination, filter-by, etc. to Entity and Instance List API. Contributed by Balu Vellanki X-Virus-Checked: Checked by ClamAV on apache.org Repository: incubator-falcon Updated Branches: refs/heads/master 85427981a -> 62b2af025 http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java index a0dbe65..7b8bdf5 100644 --- a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java +++ b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java @@ -296,8 +296,13 @@ public class SchedulableEntityManagerProxy extends AbstractSchedulableEntityMana @Produces({MediaType.TEXT_XML, MediaType.APPLICATION_JSON}) @Override public EntityList getEntityList(@PathParam("type") String type, - @DefaultValue("") @QueryParam("fields") String fields) { - return super.getEntityList(type, fields); + @DefaultValue("") @QueryParam("fields") String fields, + @DefaultValue("") @QueryParam("filterBy") String filterBy, + @DefaultValue("") @QueryParam("tags") String tags, + @DefaultValue("") @QueryParam("orderBy") String orderBy, + @DefaultValue("0") @QueryParam("offset") Integer offset, + @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) { + return super.getEntityList(type, fields, filterBy, tags, orderBy, offset, resultsPerPage); } @GET http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java ---------------------------------------------------------------------- diff --git a/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java b/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java index 73cf493..9e2d87b 100644 --- a/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java +++ b/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java @@ -18,7 +18,12 @@ package org.apache.falcon.resource; import org.apache.falcon.FalconWebException; +import org.apache.falcon.entity.v0.Entity; import org.apache.falcon.entity.v0.EntityType; +import org.apache.falcon.entity.v0.process.ACL; +import org.apache.falcon.entity.v0.process.Process; +import org.apache.falcon.security.CurrentUser; +import org.apache.falcon.util.StartupProperties; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.testng.Assert; @@ -40,7 +45,6 @@ public class EntityManagerTest extends AbstractEntityManager { @Mock private HttpServletRequest mockHttpServletRequest; - private static final String SAMPLE_PROCESS_XML = "/process-version-0.xml"; private static final String SAMPLE_INVALID_PROCESS_XML = "/process-invalid.xml"; @@ -62,12 +66,11 @@ public class EntityManagerTest extends AbstractEntityManager { /** * Run this testcase for different types of VALID entity xmls like process, feed, dataEndPoint. * - * @param stream + * @param stream entity stream * @throws IOException */ @Test(dataProvider = "validXMLServletStreamProvider") - public void testValidateForValidEntityXML(EntityType entityType, - ServletInputStream stream) throws IOException { + public void testValidateForValidEntityXML(ServletInputStream stream) throws IOException { when(mockHttpServletRequest.getInputStream()).thenReturn(stream); } @@ -102,10 +105,105 @@ public class EntityManagerTest extends AbstractEntityManager { } } + @Test + public void testGetEntityList() throws Exception { + + Entity process1 = buildProcess("processFakeUser", "fakeUser", "", ""); + configStore.publish(EntityType.PROCESS, process1); + + Entity process2 = buildProcess("processAuthUser", System.getProperty("user.name"), "", ""); + configStore.publish(EntityType.PROCESS, process2); + + EntityList entityList = this.getEntityList("process", "", "", "", "", 0, -1); + Assert.assertNotNull(entityList.getElements()); + Assert.assertEquals(entityList.getElements().length, 2); + + /* + * Both entities should be returned when the user is SuperUser. + */ + StartupProperties.get().setProperty("falcon.security.authorization.enabled", "true"); + CurrentUser.authenticate(System.getProperty("user.name")); + entityList = this.getEntityList("process", "", "", "", "", 0, -1); + Assert.assertNotNull(entityList.getElements()); + Assert.assertEquals(entityList.getElements().length, 2); + + /* + * Only one entity should be returned when the auth is enabled. + */ + CurrentUser.authenticate("fakeUser"); + entityList = this.getEntityList("process", "", "", "", "", 0, -1); + Assert.assertNotNull(entityList.getElements()); + Assert.assertEquals(entityList.getElements().length, 1); + + // reset values + StartupProperties.get().setProperty("falcon.security.authorization.enabled", "false"); + CurrentUser.authenticate(System.getProperty("user.name")); + } + + @Test + public void testGetEntityListPagination() throws Exception { + String user = System.getProperty("user.name"); + + Entity process1 = buildProcess("process1", user, + "consumer=consumer@xyz.com, owner=producer@xyz.com", + "testPipeline,dataReplicationPipeline"); + configStore.publish(EntityType.PROCESS, process1); + + Entity process2 = buildProcess("process2", user, + "consumer=consumer@xyz.com, owner=producer@xyz.com", + "testPipeline,dataReplicationPipeline"); + configStore.publish(EntityType.PROCESS, process2); + + Entity process3 = buildProcess("process3", user, "consumer=consumer@xyz.com", "testPipeline"); + configStore.publish(EntityType.PROCESS, process3); + + Entity process4 = buildProcess("process4", user, "owner=producer@xyz.com", "dataReplicationPipeline"); + configStore.publish(EntityType.PROCESS, process4); + + EntityList entityList = this.getEntityList("process", "tags", "PIPELINES:dataReplicationPipeline", + "", "name", 1, 2); + Assert.assertNotNull(entityList.getElements()); + Assert.assertEquals(entityList.getElements().length, 2); + Assert.assertEquals(entityList.getElements()[1].name, "process4"); + Assert.assertEquals(entityList.getElements()[1].tag.size(), 1); + Assert.assertEquals(entityList.getElements()[1].tag.get(0), "owner=producer@xyz.com"); + Assert.assertEquals(entityList.getElements()[0].status, null); + + + entityList = this.getEntityList("process", "pipelines", "", + "consumer=consumer@xyz.com, owner=producer@xyz.com", "name", 0, 2); + Assert.assertNotNull(entityList.getElements()); + Assert.assertEquals(entityList.getElements().length, 2); + Assert.assertEquals(entityList.getElements()[1].name, "process2"); + Assert.assertEquals(entityList.getElements()[1].pipelines.size(), 2); + Assert.assertEquals(entityList.getElements()[1].pipelines.get(0), "testPipeline"); + Assert.assertEquals(entityList.getElements()[0].tag, null); + + entityList = this.getEntityList("process", "pipelines", "", + "consumer=consumer@xyz.com, owner=producer@xyz.com", "name", 10, 2); + Assert.assertEquals(entityList.getElements().length, 0); + } + + private Entity buildProcess(String name, String username, String tags, String pipelines) { + ACL acl = new ACL(); + acl.setOwner(username); + acl.setGroup("hdfs"); + acl.setPermission("*"); + + Process p = new Process(); + p.setName(name); + p.setACL(acl); + p.setPipelines(pipelines); + p.setTags(tags); + return p; + } + + + /** * Converts a InputStream into ServletInputStream. * - * @param resourceName + * @param resourceName resource name * @return ServletInputStream */ private ServletInputStream getServletInputStream(String resourceName) { http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java b/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java index cdf703d..8762cc1 100644 --- a/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java +++ b/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java @@ -34,6 +34,7 @@ import java.util.List; @Path("instance") public class InstanceManager extends AbstractInstanceManager { + //SUSPEND CHECKSTYLE CHECK ParameterNumberCheck @GET @Path("running/{type}/{entity}") @Produces(MediaType.APPLICATION_JSON) @@ -43,8 +44,36 @@ public class InstanceManager extends AbstractInstanceManager { @Dimension("type") @PathParam("type") String type, @Dimension("entity") @PathParam("entity") String entity, @Dimension("colo") @QueryParam("colo") String colo, - @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles) { - return super.getRunningInstances(type, entity, colo, lifeCycles); + @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles, + @DefaultValue("") @QueryParam("filterBy") String filterBy, + @DefaultValue("") @QueryParam("orderBy") String orderBy, + @DefaultValue("0") @QueryParam("offset") Integer offset, + @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) { + return super.getRunningInstances(type, entity, colo, lifeCycles, filterBy, orderBy, offset, resultsPerPage); + } + + /* + getStatus(...) method actually gets all instances, filtered by a specific status. This is + a better named API which achieves the same result + */ + @GET + @Path("list/{type}/{entity}") + @Produces(MediaType.APPLICATION_JSON) + @Monitored(event = "instance-list") + @Override + public InstancesResult getInstances( + @Dimension("type") @PathParam("type") String type, + @Dimension("entity") @PathParam("entity") String entity, + @Dimension("start-time") @QueryParam("start") String startStr, + @Dimension("end-time") @QueryParam("end") String endStr, + @Dimension("colo") @QueryParam("colo") String colo, + @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles, + @DefaultValue("") @QueryParam("filterBy") String filterBy, + @DefaultValue("") @QueryParam("orderBy") String orderBy, + @DefaultValue("0") @QueryParam("offset") Integer offset, + @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) { + return super.getInstances(type, entity, startStr, endStr, colo, lifeCycles, + filterBy, orderBy, offset, resultsPerPage); } @GET @@ -52,25 +81,32 @@ public class InstanceManager extends AbstractInstanceManager { @Produces(MediaType.APPLICATION_JSON) @Monitored(event = "instance-status") @Override - public InstancesResult getStatus(@Dimension("type") @PathParam("type") String type, - @Dimension("entity") @PathParam("entity") String entity, - @Dimension("start-time") @QueryParam("start") String startStr, - @Dimension("end-time") @QueryParam("end") String endStr, - @Dimension("colo") @QueryParam("colo") String colo, - @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles) { - return super.getStatus(type, entity, startStr, endStr, colo, lifeCycles); + public InstancesResult getStatus( + @Dimension("type") @PathParam("type") String type, + @Dimension("entity") @PathParam("entity") String entity, + @Dimension("start-time") @QueryParam("start") String startStr, + @Dimension("end-time") @QueryParam("end") String endStr, + @Dimension("colo") @QueryParam("colo") String colo, + @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles, + @DefaultValue("") @QueryParam("filterBy") String filterBy, + @DefaultValue("") @QueryParam("orderBy") String orderBy, + @DefaultValue("0") @QueryParam("offset") Integer offset, + @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) { + return super.getStatus(type, entity, startStr, endStr, colo, lifeCycles, + filterBy, orderBy, offset, resultsPerPage); } @GET @Path("summary/{type}/{entity}") @Produces(MediaType.APPLICATION_JSON) @Monitored(event = "instance-summary") - public InstancesSummaryResult getSummary(@Dimension("type") @PathParam("type") String type, - @Dimension("entity") @PathParam("entity") String entity, - @Dimension("start-time") @QueryParam("start") String startStr, - @Dimension("end-time") @QueryParam("end") String endStr, - @Dimension("colo") @QueryParam("colo") String colo, - @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles) { + public InstancesSummaryResult getSummary( + @Dimension("type") @PathParam("type") String type, + @Dimension("entity") @PathParam("entity") String entity, + @Dimension("start-time") @QueryParam("start") String startStr, + @Dimension("end-time") @QueryParam("end") String endStr, + @Dimension("colo") @QueryParam("colo") String colo, + @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles) { return super.getSummary(type, entity, startStr, endStr, colo, lifeCycles); } @@ -86,8 +122,13 @@ public class InstanceManager extends AbstractInstanceManager { @Dimension("end-time") @QueryParam("end") String endStr, @Dimension("colo") @QueryParam("colo") String colo, @Dimension("run-id") @QueryParam("runid") String runId, - @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles) { - return super.getLogs(type, entity, startStr, endStr, colo, runId, lifeCycles); + @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles, + @DefaultValue("") @QueryParam("filterBy") String filterBy, + @DefaultValue("") @QueryParam("orderBy") String orderBy, + @DefaultValue("0") @QueryParam("offset") Integer offset, + @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) { + return super.getLogs(type, entity, startStr, endStr, colo, runId, lifeCycles, + filterBy, orderBy, offset, resultsPerPage); } @GET @@ -110,13 +151,14 @@ public class InstanceManager extends AbstractInstanceManager { @Produces(MediaType.APPLICATION_JSON) @Monitored(event = "kill-instance") @Override - public InstancesResult killInstance(@Context HttpServletRequest request, - @Dimension("type") @PathParam("type") String type, - @Dimension("entity") @PathParam("entity") String entity, - @Dimension("start-time") @QueryParam("start") String startStr, - @Dimension("end-time") @QueryParam("end") String endStr, - @Dimension("colo") @QueryParam("colo") String colo, - @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles) { + public InstancesResult killInstance( + @Context HttpServletRequest request, + @Dimension("type") @PathParam("type") String type, + @Dimension("entity") @PathParam("entity") String entity, + @Dimension("start-time") @QueryParam("start") String startStr, + @Dimension("end-time") @QueryParam("end") String endStr, + @Dimension("colo") @QueryParam("colo") String colo, + @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles) { return super.killInstance(request, type, entity, startStr, endStr, colo, lifeCycles); } @@ -141,13 +183,14 @@ public class InstanceManager extends AbstractInstanceManager { @Produces(MediaType.APPLICATION_JSON) @Monitored(event = "resume-instance") @Override - public InstancesResult resumeInstance(@Context HttpServletRequest request, - @Dimension("type") @PathParam("type") String type, - @Dimension("entity") @PathParam("entity") String entity, - @Dimension("start-time") @QueryParam("start") String startStr, - @Dimension("end-time") @QueryParam("end") String endStr, - @Dimension("colo") @QueryParam("colo") String colo, - @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles) { + public InstancesResult resumeInstance( + @Context HttpServletRequest request, + @Dimension("type") @PathParam("type") String type, + @Dimension("entity") @PathParam("entity") String entity, + @Dimension("start-time") @QueryParam("start") String startStr, + @Dimension("end-time") @QueryParam("end") String endStr, + @Dimension("colo") @QueryParam("colo") String colo, + @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles) { return super.resumeInstance(request, type, entity, startStr, endStr, colo, lifeCycles); } @@ -156,13 +199,16 @@ public class InstanceManager extends AbstractInstanceManager { @Produces(MediaType.APPLICATION_JSON) @Monitored(event = "re-run-instance") @Override - public InstancesResult reRunInstance(@Dimension("type") @PathParam("type") String type, - @Dimension("entity") @PathParam("entity") String entity, - @Dimension("start-time") @QueryParam("start") String startStr, - @Dimension("end-time") @QueryParam("end") String endStr, - @Context HttpServletRequest request, - @Dimension("colo") @QueryParam("colo") String colo, - @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles) { + public InstancesResult reRunInstance( + @Dimension("type") @PathParam("type") String type, + @Dimension("entity") @PathParam("entity") String entity, + @Dimension("start-time") @QueryParam("start") String startStr, + @Dimension("end-time") @QueryParam("end") String endStr, + @Context HttpServletRequest request, + @Dimension("colo") @QueryParam("colo") String colo, + @Dimension("lifecycle") @QueryParam("lifecycle") List lifeCycles) { return super.reRunInstance(type, entity, startStr, endStr, request, colo, lifeCycles); } + //RESUME CHECKSTYLE CHECK ParameterNumberCheck + } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java b/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java index 3c9078d..f993f81 100644 --- a/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java +++ b/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java @@ -59,8 +59,13 @@ public class SchedulableEntityManager extends AbstractSchedulableEntityManager { @Monitored(event = "dependencies") @Override public EntityList getEntityList(@Dimension("type") @PathParam("type") String type, - @DefaultValue("") @QueryParam("fields") String fields) { - return super.getEntityList(type, fields); + @DefaultValue("") @QueryParam("fields") String fields, + @DefaultValue("") @QueryParam("filterBy") String filterBy, + @DefaultValue("") @QueryParam("tags") String tags, + @DefaultValue("") @QueryParam("orderBy") String orderBy, + @DefaultValue("0") @QueryParam("offset") Integer offset, + @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) { + return super.getEntityList(type, fields, filterBy, tags, orderBy, offset, resultsPerPage); } @GET http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java b/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java index ea40411..b65aa46 100644 --- a/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java +++ b/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java @@ -388,6 +388,54 @@ public class FalconCLIIT { + overlay.get("processName") + " -start " + START_INSTANCE)); + // test filterBy, orderBy, offset, numResults + String startTimeString = SchemaHelper.getDateFormat().format(new Date()); + Assert.assertEquals(0, + executeWithURL("instance -running -type feed -lifecycle eviction -name " + + overlay.get("outputFeedName") + + " -start " + startTimeString + + " -orderBy startTime -offset 0 -numResults 1")); + Assert.assertEquals(-1, + executeWithURL("instance -running -type feed -lifecycle eviction -name " + + overlay.get("outputFeedName") + + " -start " + SchemaHelper.getDateFormat().format(new Date()) + + " -orderBy INVALID -offset 0 -numResults 1")); + Assert.assertEquals(-1, + executeWithURL("instance -running -type feed -lifecycle eviction -name " + + overlay.get("outputFeedName") + + " -start " + SchemaHelper.getDateFormat().format(new Date()) + + " -filterBy INVALID:FILTER -offset 0 -numResults 1")); + Assert.assertEquals(0, + executeWithURL("instance -status -type process -name " + + overlay.get("processName") + + " -start "+ START_INSTANCE + + " -filterBy STATUS:SUCCEEDED,STARTEDAFTER:"+START_INSTANCE + + " -orderBy startTime -offset 0 -numResults 1")); + Assert.assertEquals(0, + executeWithURL("instance -list -type feed -lifecycle eviction -name " + + overlay.get("outputFeedName") + + " -start "+ SchemaHelper.getDateFormat().format(new Date()) + +" -filterBy STATUS:SUCCEEDED -orderBy startTime -offset 0 -numResults 1")); + Assert.assertEquals(-1, + executeWithURL("instance -status -type feed -lifecycle eviction -name " + + overlay.get("outputFeedName") + + " -start "+ SchemaHelper.getDateFormat().format(new Date()) + +" -filterBy INVALID:FILTER -orderBy startTime -offset 0 -numResults 1")); + Assert.assertEquals(-1, + executeWithURL("instance -list -type feed -lifecycle eviction -name " + + overlay.get("outputFeedName") + + " -start "+ SchemaHelper.getDateFormat().format(new Date()) + +" -filterBy STATUS:SUCCEEDED -orderBy INVALID -offset 0 -numResults 1")); + Assert.assertEquals(0, + executeWithURL("instance -status -type feed -lifecycle eviction -name " + + overlay.get("outputFeedName") + + " -start "+ SchemaHelper.getDateFormat().format(new Date()) + +" -filterBy STATUS:SUCCEEDED -orderBy startTime -offset 1 -numResults 1")); + Assert.assertEquals(0, + executeWithURL("instance -list -type feed -lifecycle eviction -name " + + overlay.get("outputFeedName") + + " -start "+ SchemaHelper.getDateFormat().format(new Date()) + +" -filterBy STATUS:SUCCEEDED -offset 0 -numResults 1")); } public void testInstanceRunningAndSummaryCommands() throws Exception { @@ -430,7 +478,6 @@ public class FalconCLIIT { + " -start " + START_INSTANCE)); } - public void testInstanceSuspendAndResume() throws Exception { TestContext context = new TestContext(); Map overlay = context.getUniqueOverlay(); @@ -506,6 +553,86 @@ public class FalconCLIIT { + " -file "+ createTempJobPropertiesFile())); } + @Test + public void testEntityPaginationFilterByCommands() throws Exception { + + String filePath; + TestContext context = new TestContext(); + Map overlay = context.getUniqueOverlay(); + + filePath = TestContext.overlayParametersOverTemplate(context.getClusterFileTemplate(), overlay); + Assert.assertEquals(-1, + executeWithURL("entity -submitAndSchedule -type cluster -file " + filePath)); + context.setCluster(overlay.get("cluster")); + + // this is necessary for lineage + Assert.assertEquals(0, executeWithURL("entity -submit -type cluster -file " + filePath)); + + filePath = TestContext.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE1, overlay); + Assert.assertEquals(0, + executeWithURL("entity -submitAndSchedule -type feed -file " + filePath)); + + filePath = TestContext.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE2, overlay); + Assert.assertEquals(0, + executeWithURL("entity -submitAndSchedule -type feed -file " + filePath)); + + filePath = TestContext.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE1, overlay); + Assert.assertEquals(0, + executeWithURL("entity -submit -type feed -file " + filePath)); + + filePath = TestContext.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE2, overlay); + Assert.assertEquals(0, + executeWithURL("entity -submit -type feed -file " + filePath)); + + filePath = TestContext.overlayParametersOverTemplate(TestContext.PROCESS_TEMPLATE, overlay); + Assert.assertEquals(0, + executeWithURL("entity -validate -type process -file " + filePath)); + + filePath = TestContext.overlayParametersOverTemplate(TestContext.PROCESS_TEMPLATE, overlay); + Assert.assertEquals(0, + executeWithURL("entity -submitAndSchedule -type process -file " + + filePath)); + + OozieTestUtils.waitForProcessWFtoStart(context); + + // test entity List cli + Assert.assertEquals(0, + executeWithURL("entity -list -type cluster" + + " -offset 0 -numResults 1")); + Assert.assertEquals(0, + executeWithURL("entity -list -type process -fields status " + + " -filterBy STATUS:SUBMITTED,TYPE:process -orderBy name -offset 1 -numResults 1")); + Assert.assertEquals(0, + executeWithURL("entity -list -type process -fields status,pipelines " + + " -filterBy STATUS:SUBMITTED,type:process -orderBy name -offset 1 -numResults 1")); + Assert.assertEquals(0, + executeWithURL("entity -list -type process -fields status,pipelines " + + " -filterBy STATUS:SUBMITTED,pipelines:testPipeline " + + " -orderBy name -offset 1 -numResults 1")); + Assert.assertEquals(0, + executeWithURL("entity -list -type process -fields status,tags " + + " -tags owner=producer@xyz.com,department=forecasting " + + " -filterBy STATUS:SUBMITTED,type:process -orderBy name -offset 1 -numResults 1")); + Assert.assertEquals(-1, + executeWithURL("entity -list -type process -fields status " + + " -filterBy STATUS:SUCCEEDED,TYPE:process -orderBy INVALID -offset 0 -numResults 1")); + Assert.assertEquals(-1, + executeWithURL("entity -list -type process -fields INVALID " + + " -filterBy STATUS:SUCCEEDED,TYPE:process -orderBy name -offset 1 -numResults 1")); + Assert.assertEquals(-1, + executeWithURL("entity -list -type process -fields status " + + " -filterBy INVALID:FILTER,TYPE:process -orderBy name -offset 1 -numResults 1")); + Assert.assertEquals(0, + executeWithURL("entity -definition -type cluster -name " + overlay.get("cluster"))); + + Assert.assertEquals(0, + executeWithURL("instance -status -type feed -name " + + overlay.get("outputFeedName") + " -start " + START_INSTANCE)); + Assert.assertEquals(0, + executeWithURL("instance -running -type process -name " + overlay.get("processName"))); + } + + public void testContinue() throws Exception { TestContext context = new TestContext(); Map overlay = context.getUniqueOverlay(); @@ -618,6 +745,8 @@ public class FalconCLIIT { executeWithURL("entity -schedule -type feed -name " + overlay.get("outputFeedName"))); + Thread.sleep(500); + Assert.assertEquals(0, executeWithURL("instance -logs -type process -name " + overlay.get("processName") @@ -628,9 +757,31 @@ public class FalconCLIIT { + overlay.get("outputFeedName") + " -start "+ SchemaHelper.getDateFormat().format(new Date()))); + // test filterBy, orderBy, offset, numResults + Assert.assertEquals(0, + executeWithURL("instance -logs -type process -name " + + overlay.get("processName") + + " -start " + START_INSTANCE + " -end " + START_INSTANCE + + " -filterBy STATUS:SUCCEEDED -orderBy startTime -offset 0 -numResults 1")); + Assert.assertEquals(0, + executeWithURL("instance -logs -type process -name " + + overlay.get("processName") + + " -start " + START_INSTANCE + " -end " + START_INSTANCE + + " -filterBy STATUS:SUCCEEDED,STARTEDAFTER:"+START_INSTANCE+" -offset 1 -numResults 1")); + Assert.assertEquals(-1, + executeWithURL("instance -logs -type process -name " + + overlay.get("processName") + + " -start " + START_INSTANCE + " -end " + START_INSTANCE + + " -filterBy INVALID:FILTER -orderBy startTime -offset 0 -numResults 1")); + Assert.assertEquals(-1, + executeWithURL("instance -logs -type process -name " + + overlay.get("processName") + + " -start " + START_INSTANCE + " -end " + START_INSTANCE + + " -filterBy STATUS:SUCCEEDED -orderBy wrongOrder -offset 0 -numResults 1")); } private int executeWithURL(String command) throws Exception { + //System.out.println("COMMAND IS "+command + " -url " + TestContext.BASE_URL); return new FalconCLI() .run((command + " -url " + TestContext.BASE_URL).split("\\s+")); } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/test/java/org/apache/falcon/cli/FalconCLISmokeIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/cli/FalconCLISmokeIT.java b/webapp/src/test/java/org/apache/falcon/cli/FalconCLISmokeIT.java index dd9c34a..cb0dd2d 100644 --- a/webapp/src/test/java/org/apache/falcon/cli/FalconCLISmokeIT.java +++ b/webapp/src/test/java/org/apache/falcon/cli/FalconCLISmokeIT.java @@ -87,13 +87,22 @@ public class FalconCLISmokeIT { OozieTestUtils.waitForProcessWFtoStart(context); + // test entity List cli + Assert.assertEquals(0, + executeWithURL("entity -list -type cluster" + + " -offset 0 -numResults 1")); + Assert.assertEquals(0, + executeWithURL("entity -list -type process -fields status " + + " -filterBy STATUS:SUBMITTED,TYPE:process -orderBy name -offset 1 -numResults 1")); + Assert.assertEquals(-1, + executeWithURL("entity -list -type process -fields status " + + " -filterBy STATUS:SUCCEEDED,TYPE:process -orderBy INVALID -offset 0 -numResults 1")); Assert.assertEquals(0, executeWithURL("entity -definition -type cluster -name " + overlay.get("cluster"))); Assert.assertEquals(0, executeWithURL("instance -status -type feed -name " + overlay.get("outputFeedName") + " -start " + START_INSTANCE)); - Assert.assertEquals(0, executeWithURL("instance -running -type process -name " + overlay.get("processName"))); } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java b/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java index 8a50005..0a41fe0 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java +++ b/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java @@ -739,6 +739,39 @@ public class EntityManagerJerseyIT { .get(ClientResponse.class); Assert.assertEquals(response.getStatus(), 200); + EntityList result = response.getEntity(EntityList.class); + Assert.assertNotNull(result); + for (EntityList.EntityElement entityElement : result.getElements()) { + Assert.assertNull(entityElement.status); // status is null + } + + response = context.service + .path("api/entities/list/process/") + .queryParam("filterBy", "TYPE:PROCESS,STATUS:RUNNING") + .queryParam("tags", "owner=producer@xyz.com, department=forecasting") + .queryParam("orderBy", "name").queryParam("offset", "2") + .queryParam("numResults", "2").queryParam("fields", "status,tags") + .header("Cookie", context.getAuthenticationToken()) + .type(MediaType.TEXT_XML) + .accept(MediaType.TEXT_XML) + .get(ClientResponse.class); + Assert.assertEquals(response.getStatus(), 200); + result = response.getEntity(EntityList.class); + Assert.assertNotNull(result); + Assert.assertEquals(result.getElements().length, 2); + + response = context.service + .path("api/entities/list/process/") + .queryParam("orderBy", "name").queryParam("offset", "50").queryParam("numResults", "2") + .header("Cookie", context.getAuthenticationToken()) + .type(MediaType.TEXT_XML) + .accept(MediaType.TEXT_XML) + .get(ClientResponse.class); + Assert.assertEquals(response.getStatus(), 200); + result = response.getEntity(EntityList.class); + Assert.assertNotNull(result); + Assert.assertEquals(result.getElements(), null); + Map overlay = context.getUniqueOverlay(); overlay.put("cluster", "WTF-" + overlay.get("cluster")); response = context.submitToFalcon(TestContext.CLUSTER_TEMPLATE, overlay, EntityType.CLUSTER); @@ -751,7 +784,7 @@ public class EntityManagerJerseyIT { .accept(MediaType.TEXT_XML) .get(ClientResponse.class); Assert.assertEquals(response.getStatus(), 200); - EntityList result = response.getEntity(EntityList.class); + result = response.getEntity(EntityList.class); Assert.assertNotNull(result); for (EntityList.EntityElement entityElement : result.getElements()) { Assert.assertNull(entityElement.status); // status is null http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java b/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java index 03a19cd..863b109 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java +++ b/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java @@ -26,6 +26,7 @@ import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.resource.InstancesResult.Instance; import org.apache.falcon.resource.InstancesResult.WorkflowStatus; +import org.apache.falcon.security.CurrentUser; import org.apache.falcon.util.OozieTestUtils; import org.apache.falcon.workflow.engine.OozieClientFactory; import org.apache.oozie.client.ProxyOozieClient; @@ -43,10 +44,18 @@ public class ProcessInstanceManagerIT { private static final String START_INSTANCE = "2012-04-20T00:00Z"; protected void schedule(TestContext context) throws Exception { - context.scheduleProcess(); + CurrentUser.authenticate(System.getProperty("user.name")); + schedule(context, 1); + } + + protected void schedule(TestContext context, int count) throws Exception { + for (int i=0; i + consumer=consumer@xyz.com, owner=producer@xyz.com, department=forecasting + testPipeline,dataReplicationPipeline http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/test/resources/process-version-0.xml ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/process-version-0.xml b/webapp/src/test/resources/process-version-0.xml index 3ac7ac8..b193ddc 100644 --- a/webapp/src/test/resources/process-version-0.xml +++ b/webapp/src/test/resources/process-version-0.xml @@ -18,6 +18,8 @@ --> + consumer=consumer@xyz.com, owner=producer@xyz.com, department=forecasting + testPipeline,dataReplicationPipeline