falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From samar...@apache.org
Subject [2/2] falcon git commit: FALCON-843 add test to support current & last week el expression. Contributed by Pragya M
Date Fri, 27 Feb 2015 05:07:03 GMT
FALCON-843 add test to support current & last week el expression. Contributed by Pragya
M


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/833458ac
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/833458ac
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/833458ac

Branch: refs/heads/master
Commit: 833458ac10cb9b27c5f1b10641949764a3f67f6e
Parents: 444e31e
Author: samarthg <samarthg@apacge.org>
Authored: Fri Feb 27 10:35:20 2015 +0530
Committer: samarthg <samarthg@apacge.org>
Committed: Fri Feb 27 10:35:20 2015 +0530

----------------------------------------------------------------------
 falcon-regression/CHANGES.txt                   |   2 +
 .../regression/ELExpCurrentAndLastWeekTest.java | 174 +++++++++++++++++++
 2 files changed, 176 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/833458ac/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index aa33831..660ca98 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -5,6 +5,8 @@ Trunk (Unreleased)
   INCOMPATIBLE CHANGES
 
   NEW FEATURES
+   
+   FALCON-843 add test to support current & last week el expression(Pragya M via Samarth
G)
 
    FALCON-1035 Add test in falcon regression for validate feature. Validate is exposed 
    via both server and prism (Karishma G via Samarth Gupta)

http://git-wip-us.apache.org/repos/asf/falcon/blob/833458ac/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpCurrentAndLastWeekTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpCurrentAndLastWeekTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpCurrentAndLastWeekTest.java
new file mode 100644
index 0000000..7cce198
--- /dev/null
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpCurrentAndLastWeekTest.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.regression;
+
+import org.apache.falcon.regression.core.bundle.Bundle;
+import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.entity.v0.Frequency.TimeUnit;
+import org.apache.falcon.regression.core.helpers.ColoHelper;
+import org.apache.falcon.regression.core.util.*;
+import org.apache.falcon.regression.testHelper.BaseTestClass;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.log4j.Logger;
+import org.apache.oozie.client.CoordinatorAction;
+import org.apache.oozie.client.Job;
+import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.client.OozieClientException;
+import org.testng.Assert;
+import org.testng.annotations.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * EL Expression Current and last week test.
+ */
+
+@Test(groups = "embedded")
+public class ELExpCurrentAndLastWeekTest extends BaseTestClass {
+
+    private ColoHelper cluster = servers.get(0);
+    private FileSystem clusterFS = serverFS.get(0);
+    private OozieClient clusterOC = serverOC.get(0);
+    private String baseTestDir = cleanAndGetTestDir();
+    private String aggregateWorkflowDir = baseTestDir + "/aggregator";
+    private static final Logger LOGGER = Logger.getLogger(ELExpCurrentAndLastWeekTest.class);
+
+    @BeforeClass(alwaysRun = true)
+    public void createTestData() throws Exception {
+        LOGGER.info("in @BeforeClass");
+        uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
+
+        Bundle bundle = BundleUtil.readELBundle();
+        bundle.generateUniqueBundle(this);
+        bundle = new Bundle(bundle, cluster);
+
+        bundle.setInputFeedDataPath(baseTestDir + "/testData" + MINUTE_DATE_PATTERN);
+        bundle.setProcessWorkflow(aggregateWorkflowDir);
+
+    }
+
+    @BeforeMethod(alwaysRun = true)
+    public void setUp() throws Exception {
+        String processStart = "2015-02-17T10:30Z";
+        String processEnd = "2015-02-17T10:50Z";
+        bundles[0] = BundleUtil.readELBundle();
+        bundles[0] = new Bundle(bundles[0], cluster);
+        bundles[0].generateUniqueBundle(this);
+        bundles[0].setInputFeedDataPath(baseTestDir + "/testData" + MINUTE_DATE_PATTERN);
+        bundles[0].setOutputFeedLocationData(baseTestDir + "/output" + MINUTE_DATE_PATTERN);
+        bundles[0].setInputFeedPeriodicity(5, TimeUnit.minutes);
+        bundles[0].setInputFeedValidity("2010-04-01T00:00Z", "2099-04-01T00:00Z");
+        LOGGER.info("processStart: " + processStart + " processEnd: " + processEnd);
+        bundles[0].setProcessValidity(processStart, processEnd);
+        bundles[0].setProcessPeriodicity(5, TimeUnit.minutes);
+        bundles[0].setProcessWorkflow(aggregateWorkflowDir);
+    }
+
+    @AfterMethod(alwaysRun = true)
+    public void tearDown() {
+        removeTestClassEntities();
+    }
+
+    /* Test cases to check currentWeek and lastWeek EL-Expressions.
+    * Finding the missing dependencies of coordiantor based on
+    * given el definition in entity and creating them.
+    * These should match with the expected missing dependencies.
+    * In case they dont match, the test should fail.
+    *
+    * */
+    @Test(groups = {"singleCluster"}, dataProvider = "EL-DP-Cases")
+    public void currentAndLastWeekTest(String startInstance, String endInstance,
+            String firstDep, String endDep) throws Exception {
+        bundles[0].setDatasetInstances(startInstance, endInstance);
+
+        bundles[0].submitFeedsScheduleProcess(prism);
+        AssertUtil.checkStatus(clusterOC, EntityType.PROCESS, bundles[0], Job.Status.RUNNING);
+
+        InstanceUtil.waitTillInstancesAreCreated(cluster, bundles[0].getProcessData(), 0);
+
+        List<String> missingDependencies = getMissingDependencies(cluster, bundles[0]);
+        OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, bundles[0].getProcessName(),
0);
+        InstanceUtil.waitTillInstanceReachState(clusterOC, bundles[0].getProcessName(), 1,
+                CoordinatorAction.Status.SUCCEEDED, EntityType.PROCESS);
+        List<String> qaDependencies = getQADepedencyList(bundles[0], firstDep, endDep);
+        Assert.assertTrue(matchDependencies(missingDependencies, qaDependencies));
+    }
+
+    @DataProvider(name = "EL-DP-Cases")
+    public Object[][] getELData() {
+        return new Object[][]{
+            {"currentWeek('WED',2,15)", "currentWeek('WED',2,25)", "2015-02-11T02:15Z", "2015-02-11T02:25Z"},
+            {"currentWeek('WED',21,60)", "currentWeek('WED',22,10)", "2015-02-11T22:00Z",
"2015-02-11T22:10Z"},
+            {"currentWeek('WED',24,60)", "currentWeek('THU',01,10)", "2015-02-12T01:00Z",
"2015-02-12T01:10Z"},
+            {"currentWeek('WED',04,-60)", "currentWeek('WED',04,10)", "2015-02-11T03:00Z",
"2015-02-11T04:10Z"},
+            {"currentWeek('SAT',-04,-60)", "currentWeek('SAT',-04,-40)", "2015-02-13T19:00Z",
"2015-02-13T19:20Z"},
+            {"currentWeek('SAT',-24,-60)", "currentWeek('SAT',-24,-40)", "2015-02-12T23:00Z",
"2015-02-12T23:20Z"},
+            {"lastWeek('THU',-24,-20)", "lastWeek('THU',-24,-05)", "2015-02-03T23:40Z", "2015-02-03T23:55Z"},
+            {"lastWeek('WED',2,15)", "lastWeek('WED',2,25)", "2015-02-04T02:15Z", "2015-02-04T02:25Z"},
+            {"lastWeek('WED',21,60)", "lastWeek('WED',22,10)", "2015-02-04T22:00Z", "2015-02-04T22:10Z"},
+            {"lastWeek('WED',24,60)", "lastWeek('THU',01,10)", "2015-02-05T01:00Z", "2015-02-05T01:10Z"},
+            {"lastWeek('WED',04,-60)", "lastWeek('WED',04,10)", "2015-02-04T03:00Z", "2015-02-04T04:10Z"},
+            {"lastWeek('FRI',01,05)", "lastWeek('FRI',01,20)", "2015-02-06T01:05Z", "2015-02-06T01:20Z"},
+            {"lastWeek('FRI',01,60)", "lastWeek('FRI',02,20)", "2015-02-06T02:00Z", "2015-02-06T02:20Z"},
+            {"lastWeek('FRI',24,00)", "lastWeek('SAT',00,20)", "2015-02-07T00:00Z", "2015-02-07T00:20Z"},
+            {"lastWeek('THU',-04,-20)", "lastWeek('THU',-04,-05)", "2015-02-04T19:40Z", "2015-02-04T19:55Z"},
+        };
+    }
+
+    private boolean matchDependencies(List<String> fromJob, List<String> qaList)
{
+        Collections.sort(fromJob);
+        Collections.sort(qaList);
+        if (fromJob.size() != qaList.size()) {
+            return false;
+        }
+        for (int index = 0; index < fromJob.size(); index++) {
+            if (!fromJob.get(index).contains(qaList.get(index))) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private List<String> getMissingDependencies(ColoHelper prismHelper, Bundle bundle)
throws OozieClientException {
+        List<String> bundles = OozieUtil.getBundles(prismHelper.getFeedHelper().getOozieClient(),
+                Util.getProcessName(bundle.getProcessData()), EntityType.PROCESS);
+        String coordID = bundles.get(0);
+        List<String> missingDependencies =
+                OozieUtil.getMissingDependencies(prismHelper, coordID);
+        for (int i = 0; i < 10 && missingDependencies == null; ++i) {
+            TimeUtil.sleepSeconds(30);
+            missingDependencies = OozieUtil.getMissingDependencies(prismHelper, coordID);
+        }
+        Assert.assertNotNull(missingDependencies, "Missing dependencies not found.");
+        return missingDependencies;
+    }
+
+    private List<String> getQADepedencyList(Bundle bundle, String firstDep, String
endDep) {
+        String path = baseTestDir + "/testData/";
+        List<String> returnList = new ArrayList<String>();
+        List<String> dataSets = TimeUtil.getMinuteDatesOnEitherSide(firstDep,
+                endDep, bundle.getInitialDatasetFrequency());
+        for (String str : dataSets) {
+            returnList.add(path + str);
+        }
+        return returnList;
+    }
+
+}


Mime
View raw message