falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rostafiyc...@apache.org
Subject git commit: FALCON-836 Add tests for entities list API. Contributed by Ruslan Ostafiychuk
Date Wed, 29 Oct 2014 16:26:51 GMT
Repository: incubator-falcon
Updated Branches:
  refs/heads/master 20a13e3b4 -> c5c77229f


FALCON-836 Add tests for entities list API. Contributed by Ruslan Ostafiychuk


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/c5c77229
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/c5c77229
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/c5c77229

Branch: refs/heads/master
Commit: c5c77229f2b3d6989195fc532d8d323c38aef527
Parents: 20a13e3
Author: Ruslan Ostafiychuk <rostafiychuk@apache.org>
Authored: Wed Oct 29 17:47:41 2014 +0200
Committer: Ruslan Ostafiychuk <rostafiychuk@apache.org>
Committed: Wed Oct 29 18:26:10 2014 +0200

----------------------------------------------------------------------
 falcon-regression/CHANGES.txt                   |   2 +
 .../core/interfaces/IEntityManagerHelper.java   |  18 +-
 .../regression/core/response/EntityResult.java  |  40 ++-
 .../regression/core/util/CleanupUtil.java       |   2 +-
 .../regression/entity/ListEntitiesTest.java     | 299 +++++++++++++++++++
 5 files changed, 346 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c5c77229/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index f6e70e3..d5f004e 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -5,6 +5,8 @@ Trunk (Unreleased)
   INCOMPATIBLE CHANGES
 
   NEW FEATURES
+   FALCON-836 Add tests for entities list API (Ruslan Ostafiychuk)
+
    FALCON-814 Tests for entities summary API (Paul Isaychuk via Ruslan Ostafiychuk)
 
    FALCON-833 FeedReplicationTest.availabilityFlagTest fix (Paul Isaychuk

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c5c77229/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java
index 2f588ef..9c5751f 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java
@@ -255,14 +255,24 @@ public abstract class IEntityManagerHelper {
 
     public ServiceResponse listEntities()
         throws IOException, URISyntaxException, AuthenticationException {
-        return listEntities(null);
+        return listEntities(null, null);
     }
 
-    public ServiceResponse listEntities(String user)
+    public ServiceResponse listEntities(String params, String user)
         throws IOException, URISyntaxException, AuthenticationException {
         LOGGER.info("fetching " + getEntityType() + " list");
-        return Util.sendRequest(createUrl(this.hostname + URLS.LIST_URL.getValue(),
-            getEntityType() + colo), "get", null, user);
+        String url = createUrl(this.hostname + URLS.LIST_URL.getValue(),
+            getEntityType() + colo);
+        if (StringUtils.isNotEmpty(params)){
+            url += colo.isEmpty() ? "?" + params : "&" + params;
+        }
+        return Util.sendRequest(createUrl(url), "get", null, user);
+    }
+
+    public ServiceResponse listAllEntities(String params, String user)
+        throws AuthenticationException, IOException, URISyntaxException {
+        return listEntities((params == null ? "" : params + '&')
+            + "numResults=" + Integer.MAX_VALUE, user);
     }
 
     public ServiceResponse submitEntity(String data)

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c5c77229/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/EntityResult.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/EntityResult.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/EntityResult.java
index 45d4f8d..a91493f 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/EntityResult.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/EntityResult.java
@@ -18,23 +18,31 @@
 
 package org.apache.falcon.regression.core.response;
 
+import org.apache.commons.lang.StringUtils;
+
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElementWrapper;
 import javax.xml.bind.annotation.XmlRootElement;
+import java.util.List;
 
 /** Class for api result of one entity. */
 @XmlRootElement(name = "entity")
 @XmlAccessorType(XmlAccessType.FIELD)
 public class EntityResult {
-    @XmlElement(name = "type")
-    String type;
-    @XmlElement(name = "name")
-    String name;
-
-    public String getType() {
-        return type;
-    }
+    //SUSPEND CHECKSTYLE CHECK VisibilityModifierCheck
+    @XmlElement
+    public String type;
+    @XmlElement
+    public String name;
+    @XmlElement
+    public String status;
+    @XmlElementWrapper(name = "list")
+    public List<String> tag;
+    @XmlElementWrapper(name = "list")
+    public List<String> pipelines;
+    //RESUME CHECKSTYLE CHECK VisibilityModifierCheck
 
     public String getName() {
         return name;
@@ -42,7 +50,19 @@ public class EntityResult {
 
     @Override
     public String toString() {
-        return String.format("EntityResult{type='%s', name='%s'}", type, name);
-    }
+        String outString = "(" + type + ") " + name;
+        if (StringUtils.isNotEmpty(status)) {
+            outString += "(" + status + ")";
+        }
+
+        if (tag != null && !tag.isEmpty()) {
+            outString += " - " + tag.toString();
+        }
 
+        if (pipelines != null && !pipelines.isEmpty()) {
+            outString += " - " + pipelines.toString();
+        }
+        outString += "\n";
+        return outString;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c5c77229/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java
index 6b07589..87f4daa 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java
@@ -74,7 +74,7 @@ public final class CleanupUtil {
     private static EntitiesResult getEntitiesResultOfOneType(
         IEntityManagerHelper iEntityManagerHelper, String user)
         throws IOException, URISyntaxException, AuthenticationException, JAXBException {
-        final ServiceResponse clusterResponse = iEntityManagerHelper.listEntities(user);
+        final ServiceResponse clusterResponse = iEntityManagerHelper.listAllEntities(null,
user);
         JAXBContext jc = JAXBContext.newInstance(EntitiesResult.class);
         Unmarshaller u = jc.createUnmarshaller();
         return (EntitiesResult) u.unmarshal(

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c5c77229/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/entity/ListEntitiesTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/entity/ListEntitiesTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/entity/ListEntitiesTest.java
new file mode 100644
index 0000000..512fc52
--- /dev/null
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/entity/ListEntitiesTest.java
@@ -0,0 +1,299 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.regression.entity;
+
+import com.google.common.collect.Ordering;
+import org.apache.commons.lang.StringUtils;
+import org.apache.falcon.regression.Entities.ClusterMerlin;
+import org.apache.falcon.regression.Entities.FeedMerlin;
+import org.apache.falcon.regression.Entities.ProcessMerlin;
+import org.apache.falcon.regression.core.bundle.Bundle;
+import org.apache.falcon.regression.core.interfaces.IEntityManagerHelper;
+import org.apache.falcon.regression.core.response.EntityResult;
+import org.apache.falcon.regression.core.util.AssertUtil;
+import org.apache.falcon.regression.core.util.BundleUtil;
+import org.apache.falcon.regression.core.util.CleanupUtil;
+import org.apache.falcon.regression.core.util.OSUtil;
+import org.apache.falcon.regression.core.util.Util;
+import org.apache.falcon.regression.testHelper.BaseTestClass;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.log4j.Logger;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+import org.testng.asserts.SoftAssert;
+
+import javax.xml.bind.JAXBException;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Testing the list entities API
+ */
+
+public class ListEntitiesTest extends BaseTestClass {
+    private static final Logger LOGGER = Logger.getLogger(ListEntitiesTest.class);
+    private String testDir = "/ListEntitiesTest";
+    private String baseTestHDFSDir = baseHDFSDir + testDir;
+    private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator";
+    private String[] tags = {"first=yes", "second=yes", "third=yes", "wrong=no"};
+    private static final Comparator<EntityResult> NAME_COMPARATOR = new Comparator<EntityResult>()
{
+        @Override
+        public int compare(EntityResult o1, EntityResult o2) {
+            return o1.name.compareTo(o2.name);
+        }
+    };
+
+    /**
+     * Upload 10+ feeds, processes and clusters with different statuses and tags.
+     */
+    @BeforeClass(alwaysRun = true)
+    public void prepareData()
+        throws IOException, AuthenticationException, JAXBException, URISyntaxException {
+        uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
+        CleanupUtil.cleanAllEntities(prism);
+
+        bundles[0] = BundleUtil.readELBundle();
+        bundles[0] = new Bundle(bundles[0], servers.get(0));
+        bundles[0].generateUniqueBundle();
+        bundles[0].setProcessWorkflow(aggregateWorkflowDir);
+        bundles[0].submitBundle(prism);
+
+        //submit 10 different clusters, feeds and processes
+        FeedMerlin feed = new FeedMerlin(bundles[0].getInputFeedFromBundle());
+        ProcessMerlin process = new ProcessMerlin(bundles[0].getProcessData());
+        ClusterMerlin cluster = bundles[0].getClusterElement();
+        for (int i = 0; i < 10; i++) {
+            process.setName("process" + Util.getUniqueString());
+            process.setTags(getRandomTags());
+            if (i % 2 == 0) {
+                AssertUtil.assertSucceeded(prism.getProcessHelper().submitEntity(process.toString()));
+            } else {
+                AssertUtil.assertSucceeded(prism.getProcessHelper().submitAndSchedule(process.toString()));
+            }
+
+            feed.setName("feed" + Util.getUniqueString());
+            feed.setTags(getRandomTags());
+            if (i % 2 == 0) {
+                AssertUtil.assertSucceeded(prism.getFeedHelper().submitEntity(feed.toString()));
+            } else {
+                AssertUtil.assertSucceeded(prism.getFeedHelper().submitAndSchedule(feed.toString()));
+            }
+
+            cluster.setName("cluster" + Util.getUniqueString());
+            cluster.setTags(getRandomTags());
+            AssertUtil.assertSucceeded(prism.getClusterHelper().submitEntity(cluster.toString()));
+        }
+    }
+
+    @AfterClass(alwaysRun = true)
+    public void tearDown() throws IOException {
+        cleanTestDirs();
+        CleanupUtil.cleanAllEntities(prism);
+    }
+
+    /**
+     * Testing orderBy parameter. Entities should be ordered by name when orderBy=name.
+     */
+    @Test(dataProvider = "getHelpers")
+    public void listEntitiesWithOrderBy(IEntityManagerHelper helper)
+        throws AuthenticationException, IOException, URISyntaxException {
+
+        List<EntityResult> entities =
+            helper.listAllEntities("orderBy=name", null).getEntitiesResult().getEntities();
+        LOGGER.info(helper.getEntityType() + " entities: " + entities);
+        Assert.assertTrue(Ordering.from(NAME_COMPARATOR).isOrdered(entities),
+            helper.getEntityType() + " entities are not ordered by name: " + entities);
+    }
+
+
+    /**
+     * Filter entities by status (SUBMITTED or RUNNING).
+     */
+    @Test(dataProvider = "getHelpers")
+    public void listEntitiesWithFilterByStatus(IEntityManagerHelper helper)
+        throws AuthenticationException, IOException, URISyntaxException {
+        String[] statuses = helper.getEntityType().equalsIgnoreCase("cluster") ?
+            new String[]{"SUBMITTED"} : new String[]{"SUBMITTED", "RUNNING"};
+
+        List<EntityResult> allEntities =
+            helper.listAllEntities("fields=status", null).getEntitiesResult().getEntities();
+        int[] counters = new int[statuses.length];
+
+        for (EntityResult entity : allEntities) {
+            for (int i = 0; i < statuses.length; i++) {
+                if (statuses[i].equals(entity.status)) {
+                    counters[i]++;
+                }
+            }
+        }
+
+        for (int i = 0; i < statuses.length; i++) {
+            List<EntityResult> entities = helper.listAllEntities("fields=status&filterBy=STATUS:"
+                + statuses[i], null).getEntitiesResult().getEntities();
+            Assert.assertEquals(entities.size(), counters[i],
+                "Number of entities is not correct with status=" + statuses[i]);
+            for (EntityResult entity : entities) {
+                Assert.assertEquals(entity.status, statuses[i], "Entity should has status
"
+                    + statuses[i] + ". Entity: " + entity);
+            }
+
+        }
+    }
+
+    /**
+     * Testing offset parameter. Checking number of entities and order.
+     */
+    @Test(dataProvider = "getHelpers")
+    public void listEntitiesWithOffset(IEntityManagerHelper helper)
+        throws AuthenticationException, IOException, URISyntaxException {
+
+        List<EntityResult> allEntities =
+            helper.listAllEntities(null, null).getEntitiesResult().getEntities();
+        LOGGER.info(helper.getEntityType() + " entities: " + allEntities);
+        int allEntitiesCount = allEntities.size();
+        for (int i = 0; i <= allEntitiesCount; i++) {
+
+            List<EntityResult> entities =
+                helper.listEntities("offset=" + i, null).getEntitiesResult().getEntities();
+            LOGGER.info(String.format("%s entities with offset %d: %s",
+                helper.getEntityType(), i,entities));
+
+            Assert.assertEquals(entities.size(),
+                allEntitiesCount - i < 10 ? allEntitiesCount - i : 10,
+                "Number of entities is not correct.");
+            for (int j = 0; j < entities.size(); j++) {
+                Assert.assertEquals(entities.get(j).name, allEntities.get(j + i).name,
+                    "Order of entities is not correct");
+            }
+        }
+    }
+
+    /**
+     * Testing numResults parameter. Checking number of entities and order.
+     */
+    @Test(dataProvider = "getHelpers")
+    public void listEntitiesWithNumResults(IEntityManagerHelper helper)
+        throws AuthenticationException, IOException, URISyntaxException {
+
+        List<EntityResult> allEntities =
+            helper.listAllEntities(null, null).getEntitiesResult().getEntities();
+        int allEntitiesCount = allEntities.size();
+
+        for (int i = 1; i <= allEntitiesCount; i++) {
+            List<EntityResult> entities =
+                helper.listEntities("numResults=" + i, null).getEntitiesResult().getEntities();
+            Assert.assertEquals(entities.size(), i,
+                "Number of entities is not equal to numResults parameter");
+            for (int j = 0; j < i; j++) {
+                Assert.assertEquals(entities.get(j).name, allEntities.get(j).name,
+                    "Order of entities is not correct");
+            }
+        }
+
+    }
+
+    /**
+     * Get list of entities with tag.
+     */
+    @Test(dataProvider = "getHelpers")
+    public void listEntitiesWithTags(IEntityManagerHelper helper)
+        throws AuthenticationException, IOException, URISyntaxException {
+
+        List<EntityResult> allEntities =
+            helper.listAllEntities("fields=tags", null).getEntitiesResult().getEntities();
+        int[] counters = new int[tags.length];
+
+        for (EntityResult entity : allEntities) {
+            for (int i = 0; i < tags.length; i++) {
+                if (entity.tag != null && entity.tag.contains(tags[i])) {
+                   counters[i]++;
+                }
+            }
+        }
+
+        for (int i = 0; i < tags.length; i++) {
+            List<EntityResult> entities = helper.listAllEntities("fields=tags&tags="
+ tags[i],
+                null).getEntitiesResult().getEntities();
+            Assert.assertEquals(entities.size(), counters[i],
+                "Number of entities is not correct with tag=" + tags[i]);
+            for (EntityResult entity : entities) {
+                Assert.assertNotNull(entity.tag, "Entity should have tags. Entity: " + entity);
+                Assert.assertTrue(entity.tag.contains(tags[i]), "Entity should contain tag
"
+                    + tags[i] + ". Entity: " + entity);
+            }
+
+        }
+
+    }
+
+    /**
+     * Testing list entities API with custom filter.
+     */
+    @Test(dataProvider = "getHelpers")
+    public void listEntitiesWithCustomFilter(IEntityManagerHelper helper)
+        throws AuthenticationException, IOException, URISyntaxException {
+
+        List<EntityResult> entities = helper.listEntities(
+            "numResults=2&fields=status,tags&filterBy=STATUS:SUBMITTED&orderBy=name&tags="
+ tags[2],
+            null).getEntitiesResult().getEntities();
+        SoftAssert softAssert = new SoftAssert();
+        for (EntityResult entity : entities) {
+                softAssert.assertEquals(entity.status, "SUBMITTED",
+                    "Entities should have status 'SUBMITTED'");
+                softAssert.assertTrue(entity.tag.contains(tags[2]), "There is entity without
tag="
+                    + tags[2] + " Entity: " + entity);
+        }
+        softAssert.assertTrue(entities.size() <= 3, "Number of results should be 3 or
less");
+        softAssert.assertTrue(Ordering.from(NAME_COMPARATOR).isOrdered(entities),
+            helper.getEntityType() + " entities are not ordered by name: " + entities);
+        softAssert.assertAll();
+    }
+
+
+    @DataProvider
+    public Object[][] getHelpers(){
+        return new Object[][]{
+            {prism.getProcessHelper()},
+            {prism.getClusterHelper()},
+            {prism.getFeedHelper()}
+        };
+    }
+
+    private String getRandomTags() {
+        List<String> tagsList = new ArrayList<String>();
+        Random r = new Random();
+        if (r.nextInt(4) == 0) {
+            tagsList.add(tags[0]);
+        }
+        if (r.nextInt(3) == 0) {
+            tagsList.add(tags[1]);
+        }
+        if (r.nextInt(2) == 0) {
+            tagsList.add(tags[2]);
+        }
+        return StringUtils.join(tagsList, ',');
+    }
+}


Mime
View raw message