falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pisayc...@apache.org
Subject [3/3] falcon git commit: Add ProcessUpdateTest, PipelineInstanceDependencyTest and other tests and test fixes. Contributed by Raghav Gautam and Paul Isaychuk
Date Tue, 20 Oct 2015 09:37:54 GMT
Add ProcessUpdateTest, PipelineInstanceDependencyTest and other tests and test fixes. Contributed by Raghav Gautam and Paul Isaychuk


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/9e6d5a6c
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/9e6d5a6c
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/9e6d5a6c

Branch: refs/heads/master
Commit: 9e6d5a6c58d99479cc58c11b5b05b11bf8d90821
Parents: 5a55bae
Author: Paul Isaychuk <pisaychuk@apache.org>
Authored: Thu Oct 15 14:15:58 2015 +0300
Committer: Paul Isaychuk <pisaychuk@apache.org>
Committed: Tue Oct 20 11:46:31 2015 +0300

----------------------------------------------------------------------
 falcon-regression/CHANGES.txt                   |   3 +
 .../regression/Entities/ProcessMerlin.java      |   9 +
 .../helpers/entity/AbstractEntityHelper.java    |  10 +
 .../regression/core/util/EntityLineageUtil.java |  63 ++++
 .../regression/core/util/InstanceUtil.java      |  40 ++-
 .../regression/core/util/KerberosHelper.java    |   9 +
 .../falcon/regression/core/util/OozieUtil.java  |  52 +++
 .../falcon/regression/core/util/Util.java       |  22 ++
 .../ui/search/AbstractSearchPage.java           |  69 +++-
 .../regression/ui/search/ClusterWizardPage.java |  40 +--
 .../regression/ui/search/EntityWizardPage.java  |  94 ++++++
 .../regression/ui/search/FeedWizardPage.java    |  27 +-
 .../falcon/regression/ui/search/LoginPage.java  |   1 +
 .../regression/ui/search/MirrorWizardPage.java  |  13 +-
 .../falcon/regression/ui/search/PageHeader.java |  30 +-
 .../regression/ui/search/ProcessWizardPage.java |  37 +-
 .../falcon/regression/ExternalFSTest.java       |   2 +-
 .../falcon/regression/FeedReplicationTest.java  | 109 +++++-
 .../regression/ProcessInstanceStatusTest.java   |  39 +++
 .../falcon/regression/ProcessUpdateTest.java    | 112 +++++++
 .../falcon/regression/TestngListener.java       |   2 +-
 .../falcon/regression/hive/dr/HiveDRTest.java   | 118 ++++---
 .../falcon/regression/hive/dr/HiveDbDRTest.java |  61 ++--
 .../regression/hive/dr/RecipeExecLocation.java  |  63 ++++
 .../lineage/ListProcessInstancesTest.java       |  27 +-
 .../regression/searchUI/ClusterSetupTest.java   |  48 ++-
 .../regression/searchUI/FeedSetupTest.java      |  61 ++--
 .../regression/searchUI/HomePageTest.java       |   4 +-
 .../searchUI/MirrorSourceTargetOptionsTest.java |   2 +
 .../falcon/regression/searchUI/MirrorTest.java  |  76 ++++-
 .../regression/searchUI/ProcessSetupTest.java   |  65 ++--
 .../regression/security/FalconClientTest.java   |   7 +-
 .../triage/PipelineInstanceDependencyTest.java  | 335 +++++++++++++++++++
 33 files changed, 1365 insertions(+), 285 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index 5ea229a..d285661 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -5,6 +5,9 @@ Trunk (Unreleased)
   INCOMPATIBLE CHANGES
 
   NEW FEATURES
+   FALCON-1546 Add ProcessUpdateTest, PipelineInstanceDependencyTest and other tests and test fixes
+    (Raghav Gautam and Paul Isaychuk via Paul Isaychuk)
+
    FALCON-1387 Add Instance Dependency API Test(Pragya Mittal via Ajay Yadava)
 
    FALCON-1382 Add a test for feed retention to make sure that data directory is not deleted (Paul Isaychuk)

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java
index b905bee..7607aa6 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java
@@ -213,6 +213,15 @@ public class ProcessMerlin extends Process {
         return this;
     }
 
+    public String getProperty(String name) {
+        for (Property property : properties.getProperties()) {
+            if (property.getName().equals(name)) {
+                return property.getValue();
+            }
+        }
+        return null;
+    }
+
     @Override
     public String toString() {
         try {

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
index 83d06a2..e406cae 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
@@ -38,6 +38,7 @@ import org.apache.falcon.resource.FeedInstanceResult;
 import org.apache.falcon.resource.InstanceDependencyResult;
 import org.apache.falcon.resource.InstancesResult;
 import org.apache.falcon.resource.InstancesSummaryResult;
+import org.apache.falcon.resource.TriageResult;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
@@ -554,6 +555,15 @@ public abstract class AbstractEntityHelper {
     }
 
     /**
+     * Retrieves instance triage.
+     */
+    public TriageResult getInstanceTriage(String entityName, String params)
+        throws AuthenticationException, IOException, URISyntaxException, InterruptedException {
+        String url = createUrl(this.hostname + URLS.INSTANCE_TRIAGE.getValue(), getEntityType(), entityName);
+        return (TriageResult) InstanceUtil.createAndSendRequestProcessInstance(url, params, allColo, null);
+    }
+
+    /**
      * Lists all entities which are tagged by a given pipeline.
      * @param pipeline filter
      * @return service response

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/EntityLineageUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/EntityLineageUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/EntityLineageUtil.java
index fc42cf5..2df474d 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/EntityLineageUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/EntityLineageUtil.java
@@ -20,10 +20,14 @@ package org.apache.falcon.regression.core.util;
 
 import org.apache.falcon.resource.LineageGraphResult;
 import org.apache.log4j.Logger;
+import org.joda.time.DateTime;
 import org.testng.Assert;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 
@@ -34,6 +38,13 @@ public final class EntityLineageUtil{
 
     private static final Logger LOGGER = Logger.getLogger(EntityLineageUtil.class);
 
+    /**
+     * Enum to represent entity role in pipeline.
+     */
+    public enum PipelineEntityType {
+        PROCESS, INPUT_FEED, OUTPUT_FEED
+    }
+
     private EntityLineageUtil() {
         throw new AssertionError("Instantiating utility class...");
     }
@@ -61,5 +72,57 @@ public final class EntityLineageUtil{
         Assert.assertEquals(expectedVerticesSet, actualVerticesSet, "Vertices dont match");
     }
 
+    /**
+     * Produces list of expected vertices and edges in triage result.
+     */
+    public static LineageGraphResult getExpectedResult(int bundleIndx,
+                                                       Map<PipelineEntityType, List<String>> entityNamesMap,
+                                                       List<Integer> inputFeedFrequencies, String entityName,
+                                                       String clusterName, String startTime) {
+        List<String> processNames = entityNamesMap.get(PipelineEntityType.PROCESS);
+        List<String> inputFeedNames = entityNamesMap.get(PipelineEntityType.INPUT_FEED);
+        List<String> outputFeedNames = entityNamesMap.get(PipelineEntityType.OUTPUT_FEED);
+        List<String> vertices = new ArrayList<>();
+        List<LineageGraphResult.Edge> edges = new ArrayList<>();
+        final String startTimeMinus20 = TimeUtil.addMinsToTime(startTime, -20);
+        String vertexTemplate = "name: %s, type: %s, cluster: %s, instanceTime: %s, tags: %s";
+        for (int i = 0; i <= bundleIndx; ++i) {
+            //add vertex of i-th bundle process
+            boolean isTerminalInstance = processNames.contains(entityName) && i == bundleIndx;
+            String tag = isTerminalInstance ? "[WAITING]" : "Output[WAITING]";
+            final String processVertex = String.format(vertexTemplate,
+                processNames.get(i), "PROCESS", clusterName, startTime, tag);
+            vertices.add(processVertex);
+
+            //add all input feed vertices & edges for i-th bundle
+            LineageGraphResult.Edge edge;
+            String feedVertex;
+            for (DateTime dt = new DateTime(startTime); !dt.isBefore(new DateTime(startTimeMinus20));
+                 dt = dt.minusMinutes(inputFeedFrequencies.get(i))) {
+                feedVertex = String.format(vertexTemplate, inputFeedNames.get(i), "FEED",
+                    clusterName, TimeUtil.dateToOozieDate(dt.toDate()), "Input[MISSING]");
+                edge = new LineageGraphResult.Edge(feedVertex, processVertex, "consumed by");
+                vertices.add(feedVertex);
+                edges.add(edge);
+            }
+            //add output feed edge for i-th bundle
+            tag = (outputFeedNames.contains(entityName) && i == bundleIndx) ? "[MISSING]" : "Input[MISSING]";
+            feedVertex = String.format(vertexTemplate, outputFeedNames.get(i), "FEED", clusterName, startTime, tag);
+            isTerminalInstance = i == bundleIndx && outputFeedNames.contains(entityName);
+            if (i < bundleIndx || isTerminalInstance) {
+                edge = new LineageGraphResult.Edge(processVertex, feedVertex, "produces");
+                edges.add(edge);
+            }
+            //add output feed vertex only if it is terminal; it will be added as the input for next bundle otherwise
+            if (isTerminalInstance) {
+                vertices.add(feedVertex);
+            }
+        }
+        LineageGraphResult lineageGraphResult = new LineageGraphResult();
+        lineageGraphResult.setVertices(vertices.toArray(new String[vertices.size()]));
+        lineageGraphResult.setEdges(edges.toArray(new LineageGraphResult.Edge[edges.size()]));
+        return lineageGraphResult;
+    }
+
 }
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java
index 10463c2..3d05ae9 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java
@@ -36,6 +36,7 @@ import org.apache.falcon.resource.InstanceDependencyResult;
 import org.apache.falcon.resource.InstancesResult;
 import org.apache.falcon.resource.InstancesSummaryResult;
 import org.apache.falcon.resource.SchedulableEntityInstance;
+import org.apache.falcon.resource.TriageResult;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.http.HttpResponse;
@@ -96,8 +97,10 @@ public final class InstanceUtil {
             result = new InstancesSummaryResult(APIResult.Status.FAILED, responseString);
         }else if (url.contains("/listing/")) {
             result = new FeedInstanceResult(APIResult.Status.FAILED, responseString);
-        }else if (url.contains("/dependencies/")) {
+        }else if (url.contains("instance/dependencies")) {
             result = new InstanceDependencyResult(APIResult.Status.FAILED, responseString);
+        }else if (url.contains("instance/triage")) {
+            result = new TriageResult(APIResult.Status.FAILED, responseString);
         }else {
             result = new InstancesResult(APIResult.Status.FAILED, responseString);
         }
@@ -126,10 +129,7 @@ public final class InstanceUtil {
                 public Date deserialize(JsonElement json, Type t, JsonDeserializationContext c) {
                     return new DateTime(json.getAsString()).toDate();
                 }
-            }).create().fromJson(responseString,
-                    url.contains("/listing/") ? FeedInstanceResult.class : url.contains("/summary/")
-                            ? InstancesSummaryResult.class : url.contains("/dependencies/")
-                            ? InstanceDependencyResult.class : InstancesResult.class);
+            }).create().fromJson(responseString, getClassOfResult(url));
         } catch (JsonSyntaxException e) {
             Assert.fail("Not a valid json:\n" + responseString);
         }
@@ -140,6 +140,25 @@ public final class InstanceUtil {
     }
 
     /**
+     * Returns API result class matching to API request url.
+     */
+    private static Class<? extends APIResult> getClassOfResult(String url) {
+        final Class<? extends APIResult> classOfResult;
+        if (url.contains("/listing/")) {
+            classOfResult = FeedInstanceResult.class;
+        } else if (url.contains("/summary/")) {
+            classOfResult = InstancesSummaryResult.class;
+        } else if (url.contains("instance/dependencies")) {
+            classOfResult = InstanceDependencyResult.class;
+        } else if (url.contains("instance/triage")) {
+            classOfResult = TriageResult.class;
+        } else {
+            classOfResult = InstancesResult.class;
+        }
+        return classOfResult;
+    }
+
+    /**
      * Checks if API response reflects success and if it's instances match to expected status.
      *
      * @param instancesResult  - kind of response from API which should contain information about
@@ -717,8 +736,8 @@ public final class InstanceUtil {
      * @throws ParseException
      */
     public static void assertProcessInstances(InstanceDependencyResult instancesResult, OozieClient oozieClient,
-                                        String bundleID, String time) throws OozieClientException,
-                                        JSONException, ParseException {
+                                        String bundleID, String time)
+        throws OozieClientException, ParseException, JSONException {
         List<String> inputPath = new ArrayList<>();
         List<String> outputPath = new ArrayList<>();
         SchedulableEntityInstance[] instances = instancesResult.getDependencies();
@@ -808,13 +827,10 @@ public final class InstanceUtil {
      * @param processName  process name for given bundle
      * @param tag     Input/Output
      * @param expectedInstances  instance for given instanceTime.
-     * @throws JSONException
      * @throws ParseException
-     * @throws OozieClientException
      */
     public static void assertFeedInstances(InstanceDependencyResult instancesResult, String processName, String tag,
-                                            List<String> expectedInstances)
-        throws OozieClientException, JSONException, ParseException {
+                                            List<String> expectedInstances) throws ParseException {
         List<String> actualInstances = new ArrayList<>();
         SchedulableEntityInstance[] instances = instancesResult.getDependencies();
         LOGGER.info("instances: " + Arrays.toString(instances));
@@ -833,7 +849,7 @@ public final class InstanceUtil {
 
         Set<String> expectedInstancesSet = new HashSet<>(expectedInstances);
         Set<String> actualInstancesSet = new HashSet<>(actualInstances);
-        Assert.assertEquals(expectedInstancesSet, actualInstancesSet, "Instances dont match");
+        Assert.assertEquals(expectedInstancesSet, actualInstancesSet, "Instances don't match");
     }
 }
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java
index 9d028fa..c9f540f 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java
@@ -18,6 +18,7 @@
 
 package org.apache.falcon.regression.core.util;
 
+import org.apache.commons.exec.CommandLine;
 import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants;
 import org.apache.hadoop.security.UserGroupInformation;
 
@@ -41,6 +42,14 @@ public final class KerberosHelper {
             getKeyTab(user));
     }
 
+    /**
+     * Switches user in kerberos.
+     */
+    public static void initUserWithKeytab(String user){
+        ExecUtil.executeCommand(new CommandLine("sudo").addArgument("kinit").addArgument(getPrincipal(user))
+            .addArgument("-k").addArgument("-t").addArgument(getKeyTab(user)));
+    }
+
     private static String getKeyTab(String user) {
         return MerlinConstants.getKeytabForUser(user);
     }

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java
index 5e2c7b2..ae96044 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java
@@ -37,6 +37,10 @@ import org.joda.time.format.DateTimeFormat;
 import org.joda.time.format.DateTimeFormatter;
 import org.json.JSONException;
 import org.testng.Assert;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -45,6 +49,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
@@ -754,4 +759,51 @@ public final class OozieUtil {
         }
         return conf;
     }
+
+    /**
+     * Method retrieves and parses replication coordinator action workflow definition and checks whether specific
+     * properties are present in list of workflow args or not.
+     * @param workflowDefinition workflow definition
+     * @param actionName action within workflow, e.g pre-processing, replication etc.
+     * @param propMap specific properties which are expected to be in arg list
+     * @return true if all keys and values are present, false otherwise
+     */
+    public static boolean propsArePresentInWorkflow(String workflowDefinition, String actionName,
+                                              HashMap<String, String> propMap) {
+        //get action definition
+        Document definition = Util.convertStringToDocument(workflowDefinition);
+        Assert.assertNotNull(definition, "Workflow definition shouldn't be null.");
+        NodeList actions = definition.getElementsByTagName("action");
+        Element action = null;
+        for (int i = 0; i < actions.getLength(); i++) {
+            Node node = actions.item(i);
+            if (node.getNodeType() == Node.ELEMENT_NODE) {
+                action = (Element) node;
+                if (action.getAttribute("name").equals(actionName)) {
+                    break;
+                }
+                action = null;
+            }
+        }
+        Assert.assertNotNull(action, actionName + " action not found.");
+
+        //retrieve and checks whether properties are present in workflow args
+        Element javaElement = (Element) action.getElementsByTagName("java").item(0);
+        NodeList args = javaElement.getElementsByTagName("arg");
+        int counter = 0;
+        String key = null;
+        for (int i = 0; i < args.getLength(); i++) {
+            Node node = args.item(i);
+            if (node.getNodeType() == Node.ELEMENT_NODE) {
+                String argKey = node.getTextContent().replace("-", "");
+                if (key != null && propMap.get(key).equals(argKey)) {
+                    counter++;
+                    key = null;
+                } else if (key == null && propMap.containsKey(argKey)) {
+                    key = argKey;
+                }
+            }
+        }
+        return counter == propMap.size();
+    }
 }

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java
index 83547e7..ccd083b 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java
@@ -46,6 +46,7 @@ import org.joda.time.DateTime;
 import org.joda.time.format.DateTimeFormat;
 import org.joda.time.format.DateTimeFormatter;
 import org.testng.Assert;
+import org.w3c.dom.Document;
 import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
 
@@ -54,6 +55,8 @@ import javax.jms.MapMessage;
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.JAXBException;
 import javax.xml.bind.Unmarshaller;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.transform.OutputKeys;
 import javax.xml.transform.Source;
 import javax.xml.transform.Transformer;
@@ -391,6 +394,7 @@ public final class Util {
         INSTANCE_RERUN("/api/instance/rerun"),
         INSTANCE_SUMMARY("/api/instance/summary"),
         INSTANCE_PARAMS("/api/instance/params"),
+        INSTANCE_TRIAGE("/api/instance/triage"),
         INSTANCE_LIST("/api/instance/list"),
         INSTANCE_LISTING("/api/instance/listing"),
         INSTANCE_LOGS("/api/instance/logs"),
@@ -563,6 +567,24 @@ public final class Util {
     }
 
     /**
+     * Converts string to xml document.
+     * @param xmlStr string representation
+     * @return document representation.
+     */
+    public static Document convertStringToDocument(String xmlStr) {
+        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+        DocumentBuilder builder;
+        try {
+            builder = factory.newDocumentBuilder();
+            Document doc = builder.parse(new InputSource(new StringReader(xmlStr)));
+            return doc;
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+
+    /**
      * Sends api requests.
      * @param url target url
      * @param method request method

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/AbstractSearchPage.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/AbstractSearchPage.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/AbstractSearchPage.java
index d956549..ab73092 100644
--- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/AbstractSearchPage.java
+++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/AbstractSearchPage.java
@@ -20,14 +20,15 @@ package org.apache.falcon.regression.ui.search;
 
 import com.google.common.util.concurrent.SimpleTimeLimiter;
 import com.google.common.util.concurrent.TimeLimiter;
-import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.tuple.MutablePair;
 import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants;
 import org.apache.falcon.regression.core.util.TimeUtil;
 import org.apache.falcon.regression.ui.pages.Page;
 import org.apache.log4j.Logger;
 import org.openqa.selenium.By;
 import org.openqa.selenium.JavascriptExecutor;
+import org.openqa.selenium.NoSuchElementException;
 import org.openqa.selenium.TimeoutException;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.WebElement;
@@ -50,11 +51,14 @@ public abstract class AbstractSearchPage extends Page {
     public static final String UI_URL = MerlinConstants.PRISM_URL;
     private static final Logger LOGGER = Logger.getLogger(AbstractSearchPage.class);
     public static final int PAGELOAD_TIMEOUT_THRESHOLD = 10;
+    public static final int ALERT_LIFETIME = 3000;
 
     public AbstractSearchPage(WebDriver driver) {
         super(driver);
         waitForAngularToFinish();
+        LOGGER.info("Going to initialize Page Header.");
         pageHeader = PageFactory.initElements(driver, PageHeader.class);
+        LOGGER.info("Initialization done.");
     }
 
     private PageHeader pageHeader;
@@ -163,13 +167,54 @@ public abstract class AbstractSearchPage extends Page {
     public String getActiveAlertText() {
         if (waitForAlert()) {
             waitForAngularToFinish();
-            return driver.findElement(By.xpath("//div[@class='messages notifs']/div[last()]")).getText();
+            String script = "return $('div.messages.notifs > div:last-child').text();";
+            String message = (String)((JavascriptExecutor)driver).executeScript(script);
+            return message.trim();
         } else {
             return null;
         }
     }
 
     /**
+     * Wait for active alert. Check it's lifetime (the period when alert is displayed).
+     */
+    public void validateAlertLifetime() {
+        final WebElement alertsBlock = driver.findElement(By.xpath("//div[@class='messages notifs']"));
+        try {
+            final MutablePair<Long, Long> pair = new MutablePair<>(Long.MAX_VALUE, Long.MAX_VALUE);
+            // wait 5 seconds for alert to start blinking and record time of first blink
+            new WebDriverWait(driver, 5, 100).until(new ExpectedCondition<Boolean>() {
+                @Nullable
+                @Override
+                public Boolean apply(WebDriver webDriver) {
+                    String style = alertsBlock.getAttribute("style");
+                    if ((style.contains("opacity") && !style.contains("opacity: 1;"))
+                            || style.contains("display: block;")) {
+                        pair.setLeft(System.currentTimeMillis());
+                        return true;
+                    }
+                    return false;
+                }
+            });
+            // wait 5 seconds for alert to stop blinking and record time of stoppage
+            for (int i = 0; i < ALERT_LIFETIME + 3000; i += 100) {
+                String style = alertsBlock.getAttribute("style");
+                if (style.contains("display: none;")) {
+                    pair.setRight(Math.min(System.currentTimeMillis(), pair.getRight()));
+                } else {
+                    pair.setRight(Long.MAX_VALUE);
+                }
+                TimeUtil.sleepSeconds(0.1);
+            }
+            long diff = pair.getRight() - pair.getLeft();
+            LOGGER.info(String.format("Alert was live %d millis.", pair.getRight() - pair.getLeft()));
+            Assert.assertTrue(ALERT_LIFETIME <= diff, "Alert was present for too short period of time");
+        } catch (TimeoutException e) {
+            Assert.fail("Alert didn't appear in 5 seconds.");
+        }
+    }
+
+    /**
      * Wait for active alert.
      * @return true is alert is present
      */
@@ -181,12 +226,8 @@ public abstract class AbstractSearchPage extends Page {
                 @Override
                 public Boolean apply(WebDriver webDriver) {
                     String style = alertsBlock.getAttribute("style");
-                    if (style.contains("opacity") && !style.contains("opacity: 1;")) {
-                        String alert = alertsBlock.findElement(By.xpath("./div[last()]")).getText();
-                        return StringUtils.isNotEmpty(alert);
-                    } else {
-                        return false;
-                    }
+                    return (style.contains("opacity") && !style.contains("opacity: 1;"))
+                            || style.contains("display: block;");
                 }
             });
             return true;
@@ -196,6 +237,18 @@ public abstract class AbstractSearchPage extends Page {
     }
 
     /**
+     * Performs simple check of element presence.
+     */
+    public WebElement getElementOrNull(String xpath) {
+        try {
+            return driver.findElement(By.xpath(xpath));
+        } catch (NoSuchElementException ignored) {
+            return null;
+        }
+    }
+
+
+    /**
      * Method imitates click on check box. If click is not performed method retries the click.
      * @param expectedState whether check box is expected to be enabled or not after click.
      */

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ClusterWizardPage.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ClusterWizardPage.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ClusterWizardPage.java
index 0fbfc38..bcada4a 100644
--- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ClusterWizardPage.java
+++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ClusterWizardPage.java
@@ -38,7 +38,7 @@ import org.testng.Assert;
 import java.util.List;
 
 /** Page object of the Cluster creation page. */
-public class ClusterWizardPage extends AbstractSearchPage {
+public class ClusterWizardPage extends EntityWizardPage {
     private static final Logger LOGGER = Logger.getLogger(ClusterWizardPage.class);
     @FindBys({
         @FindBy(className = "mainUIView"),
@@ -53,12 +53,8 @@ public class ClusterWizardPage extends AbstractSearchPage {
     private WebElement previous;
     @FindBy(xpath = "//a[contains(text(), 'Cancel')]")
     private WebElement cancel;
-    @FindBy(id = "cluster.editXML")
-    private WebElement editXML;
     @FindBy(xpath = "//div[contains(@class, 'clusterSummaryRow')][h4]")
     private WebElement summaryBox;
-    @FindBy(xpath = "//div[contains(@class, 'xmlPreviewContainer')]//textarea")
-    private WebElement xmlPreview;
 
     public ClusterWizardPage(WebDriver driver) {
         super(driver);
@@ -273,21 +269,6 @@ public class ClusterWizardPage extends AbstractSearchPage {
         return false;
     }
 
-    public ClusterMerlin getXmlPreview() {
-        //preview block fetches changes slower then they appear on the form
-        waitForAngularToFinish();
-        String previewData = xmlPreview.getAttribute("value");
-        return new ClusterMerlin(previewData);
-    }
-
-    public void setClusterXml(String clusterXml) {
-        clickEditXml(true);
-        xmlPreview.clear();
-        xmlPreview.sendKeys(clusterXml);
-        waitForAngularToFinish();
-        clickEditXml(false);
-    }
-
     /**
      * Retrieves hte value of the summary box and parses it to cluster properties.
      * @param draft empty cluster to contain all properties.
@@ -402,16 +383,6 @@ public class ClusterWizardPage extends AbstractSearchPage {
     }
 
     /**
-     * Clicks on editXml button.
-     */
-    public void clickEditXml(boolean shouldBeEnabled) {
-        editXML.click();
-        String disabled = xmlPreview.getAttribute("disabled");
-        Assert.assertEquals(disabled == null, shouldBeEnabled,
-            "Xml preview should be " + (shouldBeEnabled ? "enabled" : "disabled"));
-    }
-
-    /**
      *  Click on next button which is the same as finish step 1.
      */
     public void clickNext() {
@@ -486,4 +457,13 @@ public class ClusterWizardPage extends AbstractSearchPage {
         }
     }
 
+    @Override
+    public WebElement getEditXMLButton() {
+        return driver.findElement(By.id("cluster.editXML"));
+    }
+
+    @Override
+    public ClusterMerlin getEntityFromXMLPreview() {
+        return new ClusterMerlin(getXMLPreview());
+    }
 }

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/EntityWizardPage.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/EntityWizardPage.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/EntityWizardPage.java
new file mode 100644
index 0000000..72c03cf
--- /dev/null
+++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/EntityWizardPage.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.regression.ui.search;
+
+import org.apache.falcon.entity.v0.Entity;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.support.FindBy;
+import org.testng.Assert;
+
+/**
+ * https://issues.apache.org/jira/browse/FALCON-1546.
+ * Parent class for cluster, feed and process wizard pages.
+ */
+public abstract class EntityWizardPage extends AbstractSearchPage {
+    @FindBy(xpath = "//i[contains(@class, 'pointer')]")
+    protected WebElement xmlPreviewPointer;
+    protected WebElement xmlPreview = null;
+
+    public EntityWizardPage(WebDriver driver) {
+        super(driver);
+    }
+
+    /**
+     * Expand/collapse xml preview.
+     * @param shouldBeExpanded should preview be expanded or collapsed.
+     */
+    public void clickXMLPreview(boolean shouldBeExpanded) {
+        if (isXmlPreviewExpanded() != shouldBeExpanded) {
+            xmlPreviewPointer.click();
+        }
+        Assert.assertEquals(isXmlPreviewExpanded(), shouldBeExpanded,
+            "Xml preview should be " + (shouldBeExpanded ? " expanded." : " collapsed."));
+    }
+
+    /**
+     * @return true if xml preview exists and is displayed, false otherwise.
+     */
+    public boolean isXmlPreviewExpanded() {
+        xmlPreview = getElementOrNull("//textarea[@ng-model='prettyXml']");
+        return xmlPreview != null && xmlPreview.isDisplayed();
+    }
+
+    public String getXMLPreview() {
+        //preview block fetches changes slower then they appear on the form
+        waitForAngularToFinish();
+        clickXMLPreview(true);
+        return xmlPreview.getAttribute("value");
+    }
+
+    public abstract Entity getEntityFromXMLPreview();
+
+    /**
+     * Pushes xml into xml preview block.
+     * @param xml entity definition
+     */
+    public void setXmlPreview(String xml) {
+        clickEditXml(true);
+        xmlPreview.clear();
+        xmlPreview.sendKeys(xml);
+        waitForAngularToFinish();
+        clickEditXml(false);
+    }
+
+    /**
+     * Clicks on editXml button.
+     */
+    public void clickEditXml(boolean shouldBeEnabled) {
+        waitForAngularToFinish();
+        clickXMLPreview(true);
+        getEditXMLButton().click();
+        String disabled = xmlPreview.getAttribute("disabled");
+        Assert.assertEquals(disabled == null, shouldBeEnabled,
+            "Xml preview should be " + (shouldBeEnabled ? "enabled" : "disabled"));
+    }
+
+    public abstract WebElement getEditXMLButton();
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/FeedWizardPage.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/FeedWizardPage.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/FeedWizardPage.java
index f3a107c..3dfab38 100644
--- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/FeedWizardPage.java
+++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/FeedWizardPage.java
@@ -34,7 +34,7 @@ import java.util.Date;
 import java.util.List;
 
 /** Page object of the Feed creation page. */
-public class FeedWizardPage extends AbstractSearchPage {
+public class FeedWizardPage extends EntityWizardPage {
 
     private static final Logger LOGGER = Logger.getLogger(FeedWizardPage.class);
 
@@ -83,17 +83,9 @@ public class FeedWizardPage extends AbstractSearchPage {
     })
     private WebElement saveFeedButton;
 
-    @FindBys({
-        @FindBy(id = "feed.editXML")
-    })
-    private WebElement editXmlButton;
-
     @FindBy(xpath = "//a[contains(.,'Cancel')]")
     private WebElement cancelButton;
 
-    @FindBy(xpath = "//textarea[@ng-model='prettyXml']")
-    private WebElement feedXml;
-
     public FeedWizardPage(WebDriver driver) {
         super(driver);
     }
@@ -327,11 +319,6 @@ public class FeedWizardPage extends AbstractSearchPage {
         cancelButton.click();
     }
 
-    public void clickEditXml(){
-        waitForAngularToFinish();
-        editXmlButton.click();
-    }
-
     public void clickCatalogStorageButton(){
         catalogStorageButton.click();
         waitForAngularToFinish();
@@ -652,14 +639,14 @@ public class FeedWizardPage extends AbstractSearchPage {
         waitForAlert();
     }
 
-    public FeedMerlin getFeedMerlinFromFeedXml() throws Exception{
-        waitForAngularToFinish();
-        return FeedMerlin.fromString(feedXml.getAttribute("value"));
+    @Override
+    public FeedMerlin getEntityFromXMLPreview() {
+        return FeedMerlin.fromString(getXMLPreview());
     }
 
-    public void setFeedXml(String xml) throws Exception{
-        feedXml.clear();
-        feedXml.sendKeys(xml);
+    @Override
+    public WebElement getEditXMLButton() {
+        return driver.findElement(By.id("feed.editXML"));
     }
 
 }

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/LoginPage.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/LoginPage.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/LoginPage.java
index 3193d21..5b261fb 100644
--- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/LoginPage.java
+++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/LoginPage.java
@@ -46,6 +46,7 @@ public class LoginPage extends AbstractSearchPage {
 
     public static LoginPage open(WebDriver driver) {
         driver.get(UI_URL);
+        LOGGER.info("Opened a URL: " + UI_URL);
         return PageFactory.initElements(driver, LoginPage.class);
     }
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/MirrorWizardPage.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/MirrorWizardPage.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/MirrorWizardPage.java
index 6dfa1ca..f990c92 100644
--- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/MirrorWizardPage.java
+++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/MirrorWizardPage.java
@@ -278,7 +278,12 @@ public class MirrorWizardPage extends AbstractSearchPage {
         return new ClusterBlock("Target");
     }
 
-    public void applyRecipe(RecipeMerlin recipe) {
+    /**
+     * Populates hive dr UI with parameters from recipe.
+     * @param recipe recipe
+     * @param overwriteDefaults should it overwrite HiveDR default values automatically picked up by UI
+     */
+    public void applyRecipe(RecipeMerlin recipe, boolean overwriteDefaults) {
         final ClusterMerlin srcCluster = recipe.getSrcCluster();
         final ClusterMerlin tgtCluster = recipe.getTgtCluster();
         setName(recipe.getName());
@@ -303,8 +308,10 @@ public class MirrorWizardPage extends AbstractSearchPage {
             setHiveReplicationMaxMaps(recipe.getReplicationMaxMaps());
             setMaxEvents(recipe.getMaxEvents());
             setHiveMaxBandwidth(recipe.getMapBandwidth());
-            setSourceInfo(recipe.getSrcCluster());
-            setTargetInfo(recipe.getTgtCluster());
+            if (overwriteDefaults) {
+                setSourceInfo(recipe.getSrcCluster());
+                setTargetInfo(recipe.getTgtCluster());
+            }
             break;
         default:
             break;

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/PageHeader.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/PageHeader.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/PageHeader.java
index 2a75b20..7f87091 100644
--- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/PageHeader.java
+++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/PageHeader.java
@@ -159,8 +159,8 @@ public class PageHeader {
             //checking if logged-in username is displayed
             if (!MerlinConstants.IS_SECURE) {
                 UIAssert.assertDisplayed(getLogoutButton(), "Logout button");
+                AssertUtil.assertNotEmpty(getLoggedInUser(), "Expecting logged-in username.");
             }
-            AssertUtil.assertNotEmpty(getLoggedInUser(), "Expecting logged-in username.");
 
             //create button navigation
             doCreateCluster();
@@ -249,7 +249,33 @@ public class PageHeader {
     }
 
     private WebElement getLogoutButton() {
-        return loginHeaderBox.findElements(By.tagName("button")).get(1);
+        return loginHeaderBox.findElements(By.xpath("button[@ng-click='logOut()']")).get(0);
+    }
+
+    private WebElement getNotificationButton() {
+        return loginHeaderBox.findElements(By.xpath("button[@ng-click='notify()']")).get(0);
+    }
+
+    /**
+     * Validates number of notifications contained by notification bar and last notification message.
+     */
+    public void validateNotificationCountAndCheckLast(int count, String message) {
+        WebElement notificationButton = getNotificationButton();
+        notificationButton.click();
+        waitForAngularToFinish();
+
+        // Test notifications dropdown visibility
+        WebElement notificationDropdown = notificationButton.findElement(By.className("messages"));
+        Assert.assertTrue(notificationDropdown.getAttribute("style").contains("display: block;"),
+            "Notifications are not visible.");
+
+        // Test validity of number of notifications
+        List<WebElement> notifications = notificationDropdown.findElements(By.xpath("div"));
+        Assert.assertEquals(notifications.size() - 1, count, "Invalid notification count.");
+
+        // Test validity of last notification
+        String lastNotification = notifications.get(0).getText();
+        Assert.assertTrue(lastNotification.contains(message), "Invalid last notification text.");
     }
 
     public LoginPage doLogout() {

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ProcessWizardPage.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ProcessWizardPage.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ProcessWizardPage.java
index 706328f..5dcd700 100644
--- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ProcessWizardPage.java
+++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ProcessWizardPage.java
@@ -32,8 +32,8 @@ import org.apache.falcon.entity.v0.process.Output;
 import org.apache.falcon.entity.v0.process.Outputs;
 import org.apache.falcon.entity.v0.process.PolicyType;
 import org.apache.falcon.entity.v0.process.Retry;
-import org.apache.falcon.entity.v0.process.Workflow;
 import org.apache.falcon.entity.v0.process.Validity;
+import org.apache.falcon.entity.v0.process.Workflow;
 import org.apache.falcon.regression.Entities.ProcessMerlin;
 import org.apache.falcon.regression.core.util.UIAssert;
 import org.apache.log4j.Logger;
@@ -52,7 +52,7 @@ import java.util.List;
 import java.util.TimeZone;
 
 /** Page object of the Process creation page. */
-public class ProcessWizardPage extends AbstractSearchPage {
+public class ProcessWizardPage extends EntityWizardPage {
 
     private static final Logger LOGGER = Logger.getLogger(ProcessWizardPage.class);
 
@@ -62,9 +62,6 @@ public class ProcessWizardPage extends AbstractSearchPage {
     })
     private WebElement processBox;
 
-    @FindBy(xpath = "//textarea[@ng-model='prettyXml']")
-    private WebElement processXml;
-
     @FindBy(xpath = "//form[@name='processForm']/div[1]")
     private WebElement summaryBox;
 
@@ -82,15 +79,10 @@ public class ProcessWizardPage extends AbstractSearchPage {
     })
     private WebElement previousButton;
 
-    @FindBys({
-        @FindBy(id = "editXmlButton")
-    })
-    private WebElement editXmlButton;
-
     @FindBy(xpath = "//a[contains(.,'Cancel')]")
     private WebElement cancelButton;
 
-    @FindBy(xpath = "//div[contains(@class,'formBoxContainer')]")
+    @FindBy(xpath = "//fieldset[@id='fieldWrapper']")
     private WebElement formBox;
 
     public ProcessWizardPage(WebDriver driver) {
@@ -130,11 +122,6 @@ public class ProcessWizardPage extends AbstractSearchPage {
         cancelButton.click();
     }
 
-    public void clickEditXml(){
-        waitForAngularToFinish();
-        editXmlButton.click();
-    }
-
     /*----- Step1 General info ----*/
 
     private WebElement getName() {
@@ -828,20 +815,14 @@ public class ProcessWizardPage extends AbstractSearchPage {
         waitForAlert();
     }
 
-    /**
-     * Creates ProcessMerlin object from xml preview string.
-     */
-    public ProcessMerlin getProcessMerlinFromProcessXml() throws Exception{
-        waitForAngularToFinish();
-        return new ProcessMerlin(processXml.getAttribute("value"));
+    @Override
+    public ProcessMerlin getEntityFromXMLPreview() {
+        return new ProcessMerlin(getXMLPreview());
     }
 
-    /**
-     * Pushes xml string to xml preview.
-     */
-    public void setProcessXml(String xml) throws Exception{
-        processXml.clear();
-        processXml.sendKeys(xml);
+    @Override
+    public WebElement getEditXMLButton() {
+        return driver.findElement(By.id("editXmlButton"));
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java
index 0662562..728b797 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java
@@ -182,9 +182,9 @@ public class ExternalFSTest extends BaseTestClass{
         Path dstPath = new Path(endpoint + testWasbTargetDir + '/' + timePattern);
 
         //check if coordinator exists
+        TimeUtil.sleepSeconds(10);
         InstanceUtil.waitTillInstancesAreCreated(clusterOC, feed.toString(), 0);
         Assert.assertEquals(OozieUtil.checkIfFeedCoordExist(clusterOC, feed.getName(), "REPLICATION"), 1);
-        TimeUtil.sleepSeconds(10);
 
         //replication should start, wait while it ends
         InstanceUtil.waitTillInstanceReachState(clusterOC, Util.readEntityName(feed.toString()), 1,

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
index 9ac9f24..6728edf 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
@@ -18,11 +18,11 @@
 
 package org.apache.falcon.regression;
 
-import org.apache.falcon.regression.Entities.FeedMerlin;
-import org.apache.falcon.regression.core.bundle.Bundle;
 import org.apache.falcon.entity.v0.EntityType;
 import org.apache.falcon.entity.v0.feed.ActionType;
 import org.apache.falcon.entity.v0.feed.ClusterType;
+import org.apache.falcon.regression.Entities.FeedMerlin;
+import org.apache.falcon.regression.core.bundle.Bundle;
 import org.apache.falcon.regression.core.helpers.ColoHelper;
 import org.apache.falcon.regression.core.util.AssertUtil;
 import org.apache.falcon.regression.core.util.BundleUtil;
@@ -36,9 +36,11 @@ import org.apache.falcon.regression.testHelper.BaseTestClass;
 import org.apache.falcon.resource.InstancesResult;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.log4j.Logger;
 import org.apache.oozie.client.CoordinatorAction;
 import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.client.OozieClientException;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
@@ -51,7 +53,10 @@ import org.testng.annotations.Test;
 
 import javax.xml.bind.JAXBException;
 import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * feed replication test.
@@ -383,6 +388,106 @@ public class FeedReplicationTest extends BaseTestClass {
     }
 
     /**
+     * Test for https://issues.apache.org/jira/browse/FALCON-668.
+     * Check that new DistCp options are allowed.
+     */
+    @Test
+    public void testNewDistCpOptions()
+        throws URISyntaxException, AuthenticationException, InterruptedException, IOException, JAXBException,
+        OozieClientException {
+        Bundle.submitCluster(bundles[0], bundles[1]);
+        String startTime = TimeUtil.getTimeWrtSystemTime(0);
+        String endTime = TimeUtil.addMinsToTime(startTime, 5);
+        LOGGER.info("Time range between : " + startTime + " and " + endTime);
+        //configure feed
+        String feedName = Util.readEntityName(bundles[0].getDataSets().get(0));
+        FeedMerlin feedElement = bundles[0].getFeedElement(feedName);
+        bundles[0].writeFeedElement(feedElement, feedName);
+        FeedMerlin feed = new FeedMerlin(bundles[0].getDataSets().get(0));
+        feed.setFilePath(feedDataLocation);
+        //erase all clusters from feed definition
+        feed.clearFeedClusters();
+        //set cluster1 as source
+        feed.addFeedCluster(
+            new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0)))
+                .withRetention("days(1000000)", ActionType.DELETE)
+                .withValidity(startTime, endTime)
+                .withClusterType(ClusterType.SOURCE)
+                .build());
+        //set cluster2 as target
+        feed.addFeedCluster(
+            new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[1].getClusters().get(0)))
+                .withRetention("days(1000000)", ActionType.DELETE)
+                .withValidity(startTime, endTime)
+                .withClusterType(ClusterType.TARGET)
+                .withDataLocation(targetDataLocation)
+                .build());
+
+        //add custom properties to feed
+        HashMap<String, String> propMap = new HashMap<>();
+        propMap.put("overwrite", "true");
+        propMap.put("ignoreErrors", "false");
+        propMap.put("skipChecksum", "false");
+        propMap.put("removeDeletedFiles", "true");
+        propMap.put("preserveBlockSize", "true");
+        propMap.put("preserveReplicationNumber", "true");
+        propMap.put("preservePermission", "true");
+        for (Map.Entry<String, String> entry : propMap.entrySet()) {
+            feed.withProperty(entry.getKey(), entry.getValue());
+        }
+        //add custom property which shouldn't be passed to workflow
+        HashMap<String, String> unsupportedPropMap = new HashMap<>();
+        unsupportedPropMap.put("myCustomProperty", "true");
+        feed.withProperty("myCustomProperty", "true");
+
+        //upload necessary data to source
+        DateTime date = new DateTime(startTime, DateTimeZone.UTC);
+        DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy'/'MM'/'dd'/'HH'/'mm'");
+        String timePattern = fmt.print(date);
+        String sourceLocation = sourcePath + "/" + timePattern + "/";
+        HadoopUtil.recreateDir(cluster1FS, sourceLocation);
+        HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile.xml"));
+        HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.concat(OSUtil.NORMAL_INPUT,  "dataFile1.txt"));
+
+        //copy 2 files to target to check if they will be deleted because of removeDeletedFiles property
+        String targetLocation = targetPath + "/" + timePattern + "/";
+        cluster2FS.copyFromLocalFile(new Path(OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile3.txt")),
+            new Path(targetLocation + "dataFile3.txt"));
+
+        //submit and schedule feed
+        LOGGER.info("Feed : " + Util.prettyPrintXml(feed.toString()));
+        AssertUtil.assertSucceeded(prism.getFeedHelper().submitAndSchedule(feed.toString()));
+
+        //check while instance is got created
+        InstanceUtil.waitTillInstancesAreCreated(cluster2OC, feed.toString(), 0);
+
+        //check if coordinator exists and replication starts
+        Assert.assertEquals(OozieUtil.checkIfFeedCoordExist(cluster2OC, feed.getName(), "REPLICATION"), 1);
+        InstanceUtil.waitTillInstanceReachState(cluster2OC, feed.getName(), 1,
+            CoordinatorAction.Status.RUNNING, EntityType.FEED);
+
+        //check that properties were passed to workflow definition
+        String bundleId = OozieUtil.getLatestBundleID(cluster2OC, feedName, EntityType.FEED);
+        String coordId = OozieUtil.getReplicationCoordID(bundleId, cluster2.getFeedHelper()).get(0);
+        CoordinatorAction coordinatorAction = cluster2OC.getCoordJobInfo(coordId).getActions().get(0);
+        String wfDefinition = cluster2OC.getJobDefinition(coordinatorAction.getExternalId());
+        LOGGER.info(String.format("Definition of coordinator job action %s : \n %s \n",
+            coordinatorAction.getExternalId(), Util.prettyPrintXml(wfDefinition)));
+        Assert.assertTrue(OozieUtil.propsArePresentInWorkflow(wfDefinition, "replication", propMap),
+            "New distCp supported properties should be passed to replication args list.");
+        Assert.assertFalse(OozieUtil.propsArePresentInWorkflow(wfDefinition, "replication", unsupportedPropMap),
+            "Unsupported properties shouldn't be passed to replication args list.");
+
+        //check that replication succeeds
+        InstanceUtil.waitTillInstanceReachState(cluster2OC, feed.getName(), 1,
+            CoordinatorAction.Status.SUCCEEDED, EntityType.FEED);
+
+        List<Path> finalFiles = HadoopUtil.getAllFilesRecursivelyHDFS(cluster2FS, new Path(targetPath));
+        Assert.assertEquals(finalFiles.size(), 2, "Only replicated files should be present on target "
+            + "because of 'removeDeletedFiles' distCp property.");
+    }
+
+    /**
      * Test demonstrates failure pf replication of stored data from one source cluster to one target cluster.
      * When replication job fails test checks if failed logs are present in staging directory or not.
      */

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java
index 7f1e445..6493133 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java
@@ -434,6 +434,45 @@ public class ProcessInstanceStatusTest extends BaseTestClass {
         InstanceUtil.validateFailedInstances(r, 3);
     }
 
+    /**
+     * Check that default end time param value is now.
+     */
+    @Test
+    public void testDefaultEndTimeParam()
+        throws OozieClientException, IOException, InterruptedException, AuthenticationException, URISyntaxException,
+        JAXBException {
+        //set validity to have 12 instances
+        String start = TimeUtil.getTimeWrtSystemTime(-60);
+        String end = TimeUtil.getTimeWrtSystemTime(0);
+        bundles[0].setProcessValidity(start, end);
+        bundles[0].setProcessPeriodicity(5, TimeUnit.minutes);
+        bundles[0].setProcessConcurrency(3);
+        bundles[0].submitFeedsScheduleProcess(prism);
+        InstanceUtil.waitTillInstancesAreCreated(clusterOC, bundles[0].getProcessData(), 0);
+        //make first 3 instances running
+        OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 0);
+        OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 1);
+        OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 2);
+        InstanceUtil.waitTillInstanceReachState(clusterOC, processName, 3, Status.RUNNING,
+            EntityType.PROCESS);
+        //check instances status with end, expected first 10 instances
+        InstancesResult r = prism.getProcessHelper().getProcessInstanceStatus(processName,
+            "?start=" + start + "&end=" + TimeUtil.addMinsToTime(end, -11));
+        InstanceUtil.validateResponse(r, 10, 3, 0, 7, 0);
+        //request the same but without end, expected to have the latest 10 instances
+        r = prism.getProcessHelper().getProcessInstanceStatus(processName,
+            "?start=" + start);
+        InstanceUtil.validateResponse(r, 10, 1, 0, 9, 0);
+        //the same with numResults which includes/excludes all running instances
+        r = prism.getProcessHelper().getProcessInstanceStatus(processName,
+            "?start=" + start + "&end=" + TimeUtil.addMinsToTime(end, -16) + "&numResults=9");
+        InstanceUtil.validateResponse(r, 9, 3, 0, 6, 0);
+        //expected end is set to now, thus getting last 9 instances
+        r = prism.getProcessHelper().getProcessInstanceStatus(processName,
+            "?start=" + start + "&numResults=9");
+        InstanceUtil.validateResponse(r, 9, 0, 0, 9, 0);
+    }
+
     /*
     * Function to match the workflows obtained from instance status and oozie.
     */

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessUpdateTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessUpdateTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessUpdateTest.java
new file mode 100644
index 0000000..efbb503
--- /dev/null
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessUpdateTest.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.regression;
+
+import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.entity.v0.Frequency;
+import org.apache.falcon.entity.v0.process.LateInput;
+import org.apache.falcon.entity.v0.process.LateProcess;
+import org.apache.falcon.entity.v0.process.PolicyType;
+import org.apache.falcon.regression.Entities.ProcessMerlin;
+import org.apache.falcon.regression.core.bundle.Bundle;
+import org.apache.falcon.regression.core.helpers.ColoHelper;
+import org.apache.falcon.regression.core.util.AssertUtil;
+import org.apache.falcon.regression.core.util.BundleUtil;
+import org.apache.falcon.regression.core.util.InstanceUtil;
+import org.apache.falcon.regression.core.util.OSUtil;
+import org.apache.falcon.regression.core.util.OozieUtil;
+import org.apache.falcon.regression.core.util.TimeUtil;
+import org.apache.falcon.regression.core.util.Util;
+import org.apache.falcon.regression.testHelper.BaseTestClass;
+import org.apache.log4j.Logger;
+import org.apache.oozie.client.OozieClient;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Tests related to update feature.
+ */
+@Test(groups = "embedded")
+public class ProcessUpdateTest extends BaseTestClass {
+
+    private ColoHelper cluster = servers.get(0);
+    private OozieClient clusterOC = serverOC.get(0);
+    private String baseTestHDFSDir = cleanAndGetTestDir();
+    private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator";
+    private String feedInputPath = baseTestHDFSDir + "/input" + MINUTE_DATE_PATTERN;
+    private String feedOutputPath = baseTestHDFSDir + "/output-data" + MINUTE_DATE_PATTERN;
+    private static final Logger LOGGER = Logger.getLogger(ProcessUpdateTest.class);
+
+    @BeforeClass(alwaysRun = true)
+    public void uploadWorkflow() throws Exception {
+        uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
+    }
+
+    @BeforeMethod(alwaysRun = true)
+    public void setUp() throws Exception {
+        Bundle bundle = BundleUtil.readELBundle();
+        bundles[0] = new Bundle(bundle, servers.get(0));
+        bundles[0].generateUniqueBundle(this);
+        bundles[0].setProcessWorkflow(aggregateWorkflowDir);
+        bundles[0].setInputFeedDataPath(feedInputPath);
+        bundles[0].setOutputFeedLocationData(feedOutputPath);
+    }
+
+    /**
+     * Test for https://issues.apache.org/jira/browse/FALCON-99.
+     * Scenario: schedule a process which doesn't have late data handling and then update it to have it.
+     * Check that new coordinator was created.
+     */
+    @Test
+    public void updateProcessWithLateData() throws Exception {
+        String start = TimeUtil.getTimeWrtSystemTime(-60);
+        String end = TimeUtil.getTimeWrtSystemTime(10);
+        bundles[0].submitAndScheduleAllFeeds();
+        ProcessMerlin process = bundles[0].getProcessObject();
+        process.setValidity(start, end);
+        process.setLateProcess(null);
+        cluster.getProcessHelper().submitAndSchedule(process.toString());
+        InstanceUtil.waitTillInstancesAreCreated(clusterOC, process.toString(), 0);
+        String bundleId = OozieUtil.getLatestBundleID(clusterOC, process.getName(), EntityType.PROCESS);
+
+        //update process to have late data handling
+        LateProcess lateProcess = new LateProcess();
+        lateProcess.setDelay(new Frequency("hours(1)"));
+        lateProcess.setPolicy(PolicyType.EXP_BACKOFF);
+        LateInput lateInput = new LateInput();
+        lateInput.setInput("inputData");
+        lateInput.setWorkflowPath(aggregateWorkflowDir);
+        lateProcess.getLateInputs().add(lateInput);
+        process.setLateProcess(lateProcess);
+        LOGGER.info("Updated process xml: " + Util.prettyPrintXml(process.toString()));
+        AssertUtil.assertSucceeded(cluster.getProcessHelper().update(process.toString(), process.toString()));
+
+        //check that new coordinator was created
+        String newBundleId = OozieUtil.getLatestBundleID(clusterOC, process.getName(), EntityType.PROCESS);
+        Assert.assertNotEquals(bundleId, newBundleId, "New Bundle should be created.");
+    }
+
+    @AfterMethod(alwaysRun = true)
+    public void tearDown() {
+        removeTestClassEntities();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java
index 9ea8471..ca6ee88 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java
@@ -77,7 +77,7 @@ public class TestngListener implements ITestListener, IExecutionListener {
             LOGGER.info("Dumping of falcon store failed: " + e);
         }
         LOGGER.info(
-            String.format("Testing going to end for: %s.%s(%s) %s", result.getTestClass().getName(),
+            String.format("Testing going to end for: %s.%s(%s) ----- Status: %s", result.getTestClass().getName(),
                 result.getName(), Arrays.toString(result.getParameters()), outcome));
         NDC.pop();
         LOGGER.info(hr);


Mime
View raw message