falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From samar...@apache.org
Subject [1/3] falcon git commit: FALCON-1043 Add test in falcon to test touch feature. Contributed by Pragya M.
Date Wed, 25 Feb 2015 05:40:20 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 65cb71841 -> 9dd0b27e5


FALCON-1043 Add test in falcon to test touch feature. Contributed by Pragya M.


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/87bff1fc
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/87bff1fc
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/87bff1fc

Branch: refs/heads/master
Commit: 87bff1fc111f0fb58324db82a6734fa73cefcbc1
Parents: 65cb718
Author: samarthg <samarthg@apacge.org>
Authored: Wed Feb 25 10:51:17 2015 +0530
Committer: samarthg <samarthg@apacge.org>
Committed: Wed Feb 25 10:51:17 2015 +0530

----------------------------------------------------------------------
 falcon-regression/CHANGES.txt                   |   2 +
 .../helpers/entity/AbstractEntityHelper.java    |  12 ++
 .../falcon/regression/core/util/Util.java       |   4 +-
 .../regression/TouchAPIPrismAndServerTest.java  | 183 +++++++++++++++++++
 4 files changed, 200 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/87bff1fc/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index 4ad8a2a..fc3ae50 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -6,6 +6,8 @@ Trunk (Unreleased)
 
   NEW FEATURES
 
+   FALCON-1043 Add test in falcon to test touch feature(Pragya Mittal via Samarth Gupta)
  
+
    FALCON-1021 Add test in falcon to test feed SLA feature (Pragya Mittal via Raghav Kumar
Gautam)
 
    FALCON-671 Add a test in falcon-regression where one oozie workflow has various actions
like

http://git-wip-us.apache.org/repos/asf/falcon/blob/87bff1fc/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
index e11b37d..3828b2f 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
@@ -597,4 +597,16 @@ public abstract class AbstractEntityHelper {
             entityName + colo);
         return Util.sendRequest(url, "get", null, null);
     }
+
+    public ServiceResponse touchEntity(String data)
+        throws IOException, URISyntaxException, AuthenticationException, InterruptedException
{
+        return touchEntity(Util.readEntityName(data), data, null);
+    }
+
+    public ServiceResponse touchEntity(String entityName, String data, String user)
+        throws AuthenticationException, IOException, URISyntaxException, InterruptedException
{
+        String url = createUrl(this.hostname + URLS.TOUCH_URL.getValue(), getEntityType(),
+                entityName + colo);
+        return Util.sendRequest(url, "post", data, user);
+    }
 }

http://git-wip-us.apache.org/repos/asf/falcon/blob/87bff1fc/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java
index 5717923..ab09ff1 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java
@@ -596,7 +596,9 @@ public final class Util {
         INSTANCE_RERUN("/api/instance/rerun"),
         INSTANCE_SUMMARY("/api/instance/summary"),
         INSTANCE_PARAMS("/api/instance/params"),
-        INSTANCE_LIST("/api/instance/list");
+        INSTANCE_LIST("/api/instance/list"),
+        TOUCH_URL("/api/entities/touch");
+
         private final String url;
 
         URLS(String url) {

http://git-wip-us.apache.org/repos/asf/falcon/blob/87bff1fc/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java
new file mode 100644
index 0000000..1f3d990
--- /dev/null
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.regression;
+
+import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.entity.v0.Frequency;
+import org.apache.falcon.regression.core.bundle.Bundle;
+import org.apache.falcon.regression.core.helpers.ColoHelper;
+import org.apache.falcon.regression.core.response.ServiceResponse;
+import org.apache.falcon.regression.core.util.*;
+import org.apache.falcon.regression.testHelper.BaseTestClass;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.log4j.Logger;
+import org.apache.oozie.client.Job;
+import org.apache.oozie.client.OozieClient;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import javax.xml.bind.JAXBException;
+
+/**
+ * Touch feature test both via server and prism.
+ */
+public class TouchAPIPrismAndServerTest extends BaseTestClass {
+    private ColoHelper cluster = servers.get(0);
+    private OozieClient clusterOC = serverOC.get(0);
+    private FileSystem clusterFS = serverFS.get(0);
+    private String aggregateWorkflowDir = cleanAndGetTestDir() + "/aggregator";
+    private String feed;
+    private static final Logger LOGGER = Logger.getLogger(TouchAPIPrismAndServerTest.class);
+    private String startTime;
+    private String endTime;
+
+    @BeforeClass(alwaysRun = true)
+    public void uploadWorkflow() throws Exception {
+        uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
+    }
+
+    @BeforeMethod(alwaysRun = true)
+    public void setUp() throws Exception {
+        startTime = TimeUtil.getTimeWrtSystemTime(0);
+        endTime = TimeUtil.addMinsToTime(startTime, 20);
+        LOGGER.info("Time range between : " + startTime + " and " + endTime);
+        bundles[0] = BundleUtil.readELBundle();
+        bundles[0] = new Bundle(bundles[0], cluster);
+        bundles[0].generateUniqueBundle(this);
+        bundles[0].setProcessWorkflow(aggregateWorkflowDir);
+        bundles[0].setProcessValidity(startTime, endTime);
+        bundles[0].setProcessPeriodicity(5, Frequency.TimeUnit.minutes);
+        bundles[0].setOutputFeedPeriodicity(5, Frequency.TimeUnit.minutes);
+    }
+
+    @AfterMethod(alwaysRun = true)
+    public void tearDown() {
+        removeTestClassEntities();
+    }
+
+    /**
+     * Schedule a process with end time greater than current time
+     * Perform touch via both server and prism
+     * Should succeed with creation of a new bundle from the current time.
+     *
+     * @throws Exception
+     */
+    @Test(groups = {"singleCluster"})
+    public void touchProcessSchedule() throws Exception {
+        bundles[0].submitFeedsScheduleProcess(prism);
+        AssertUtil.checkStatus(clusterOC, EntityType.PROCESS, bundles[0], Job.Status.RUNNING);
+        InstanceUtil.waitTillInstancesAreCreated(cluster, bundles[0].getProcessData(), 0);
+        String coordId = InstanceUtil.getLatestCoordinatorID(clusterOC,
+                bundles[0].getProcessName(), EntityType.PROCESS);
+        String oldbundleId = InstanceUtil.getLatestBundleID(cluster, bundles[0].getProcessName(),
EntityType.PROCESS);
+
+        // via prism
+        ServiceResponse response = prism.getProcessHelper().touchEntity(bundles[0].getProcessData());
+        String bundleId = InstanceUtil.getLatestBundleID(cluster, bundles[0].getProcessName(),
EntityType.PROCESS);
+        Assert.assertNotEquals(oldbundleId, bundleId, "Bundle ids are same. No new bundle
generated.");
+        validate(response, "Old bundle id: " + coordId + ". New bundle id: " + bundleId);
+
+        // via server
+        oldbundleId = bundleId;
+        coordId = InstanceUtil.getLatestCoordinatorID(clusterOC, bundles[0].getProcessName(),
EntityType.PROCESS);
+        response = cluster.getProcessHelper().touchEntity(bundles[0].getProcessData());
+        bundleId = InstanceUtil.getLatestBundleID(cluster, bundles[0].getProcessName(), EntityType.PROCESS);
+        Assert.assertNotEquals(oldbundleId, bundleId, "Bundle ids are same. No new bundle
generated.");
+        validate(response, "Old bundle id: " + coordId + ". New bundle id: " + bundleId);
+    }
+
+    /**
+     * Schedule a feed with end time greater than current time
+     * Perform touch via both server and prism
+     * Should succeed with creation of a new bundle from the current time.
+     *
+     * @throws Exception
+     */
+    @Test(groups = {"singleCluster"})
+    public void touchFeedSchedule() throws Exception {
+        bundles[0].submitAndScheduleFeed();
+        AssertUtil.checkStatus(clusterOC, EntityType.FEED, bundles[0], Job.Status.RUNNING);
+        String coordId = InstanceUtil.getLatestCoordinatorID(clusterOC,
+                Util.readEntityName(bundles[0].getDataSets().get(0)), EntityType.FEED);
+        String oldbundleId = InstanceUtil.getLatestBundleID(cluster,
+                Util.readEntityName(bundles[0].getDataSets().get(0)), EntityType.FEED);
+
+        // via prism
+        TimeUtil.sleepSeconds(60);
+        ServiceResponse response = prism.getFeedHelper().touchEntity(bundles[0].getDataSets().get(0));
+        String bundleId = InstanceUtil.getLatestBundleID(cluster, Util.readEntityName(bundles[0].getDataSets().get(0)),
+                EntityType.FEED);
+        Assert.assertNotEquals(oldbundleId, bundleId, "Bundle ids are same. No new bundle
generated.");
+        validate(response, "Old bundle id: " + coordId + ". New bundle id: " + bundleId);
+
+        // via server
+        oldbundleId = bundleId;
+        coordId = InstanceUtil.getLatestCoordinatorID(clusterOC, Util.readEntityName(bundles[0].getDataSets().get(0)),
+                EntityType.FEED);
+        TimeUtil.sleepSeconds(60);
+        response = cluster.getFeedHelper().touchEntity(bundles[0].getDataSets().get(0));
+        bundleId = InstanceUtil.getLatestBundleID(cluster, Util.readEntityName(bundles[0].getDataSets().get(0)),
+                EntityType.FEED);
+        Assert.assertNotEquals(oldbundleId, bundleId, "Bundle ids are same. No new bundle
generated.");
+        validate(response, "Old bundle id: " + coordId + ". New bundle id: " + bundleId);
+
+    }
+
+    /**
+     * Schedule a process with end time less than current time.
+     * Perform touch via both server and prism
+     * No new bundle should be created.
+     *
+     * @throws Exception
+     */
+    @Test(groups = {"singleCluster"})
+    public void touchProcessScheduleWithEndTimeLessThanCurrentTime() throws Exception {
+        startTime = TimeUtil.getTimeWrtSystemTime(-120);
+        endTime = TimeUtil.addMinsToTime(startTime, 20);
+        bundles[0].setProcessValidity(startTime, endTime);
+        bundles[0].submitFeedsScheduleProcess(prism);
+        AssertUtil.checkStatus(clusterOC, EntityType.PROCESS, bundles[0], Job.Status.RUNNING);
+        InstanceUtil.waitTillInstancesAreCreated(cluster, bundles[0].getProcessData(), 0);
+        String coordId = InstanceUtil.getLatestCoordinatorID(clusterOC,
+                bundles[0].getProcessName(), EntityType.PROCESS);
+        String oldbundleId = InstanceUtil.getLatestBundleID(cluster, bundles[0].getProcessName(),
EntityType.PROCESS);
+
+        // via prism
+        ServiceResponse response = prism.getProcessHelper().touchEntity(bundles[0].getProcessData());
+        String bundleId = InstanceUtil.getLatestBundleID(cluster, bundles[0].getProcessName(),
EntityType.PROCESS);
+        Assert.assertEquals(oldbundleId, bundleId, "New bundle generated");
+        validate(response, "Old bundle id: " + coordId);
+
+        //via server
+        oldbundleId = bundleId;
+        response = cluster.getProcessHelper().touchEntity(bundles[0].getProcessData());
+        bundleId = InstanceUtil.getLatestBundleID(cluster, bundles[0].getProcessName(), EntityType.PROCESS);
+        Assert.assertEquals(oldbundleId, bundleId, "New bundle generated");
+        validate(response, "Old bundle id: " + coordId);
+    }
+
+    private void validate(ServiceResponse response, String message) throws JAXBException
{
+        AssertUtil.assertSucceeded(response);
+        Assert.assertTrue(response.getMessage().contains(message),
+                "Correct response was not present in process / feed schedule");
+    }
+}


Mime
View raw message