falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prag...@apache.org
Subject falcon git commit: FALCON-1699 Test fixes for RetentionTest, LineageApiTest, TouchAPIPrismAndServerTest, FeedReplicationTest and few fortifications. Contributed by Paul Isaychuk.
Date Mon, 11 Jan 2016 10:39:16 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 4656f692a -> e9e0c7b7c


FALCON-1699 Test fixes for RetentionTest, LineageApiTest, TouchAPIPrismAndServerTest, FeedReplicationTest
and few fortifications. Contributed by Paul Isaychuk.


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/e9e0c7b7
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/e9e0c7b7
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/e9e0c7b7

Branch: refs/heads/master
Commit: e9e0c7b7c0aed12e36c61f2cf8b4e67458fdfd06
Parents: 4656f69
Author: Pragya <mittal.pragya23@gmail.com>
Authored: Mon Jan 11 10:30:06 2016 +0000
Committer: Pragya <mittal.pragya23@gmail.com>
Committed: Mon Jan 11 10:30:06 2016 +0000

----------------------------------------------------------------------
 falcon-regression/CHANGES.txt                   |  2 +
 .../regression/core/response/lineage/Edge.java  |  3 ++
 .../regression/core/util/KerberosHelper.java    |  9 ----
 .../falcon/regression/core/util/OozieUtil.java  | 52 +++++++++++++-------
 .../org/apache/falcon/request/BaseRequest.java  |  3 ++
 .../regression/TouchAPIPrismAndServerTest.java  |  1 +
 .../regression/hcat/HCatRetentionTest.java      |  9 +++-
 .../lineage/ListFeedInstancesTest.java          | 12 ++---
 .../falcon/regression/prism/RetentionTest.java  |  6 ++-
 .../regression/security/FalconClientTest.java   |  7 +--
 10 files changed, 63 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index a4ae687..f2df91a 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -5,6 +5,8 @@ Trunk (Unreleased)
   INCOMPATIBLE CHANGES
 
   NEW FEATURES
+   FALCON-1699 Test fixes for RetentionTest, LineageApiTest, TouchAPIPrismAndServerTest,
FeedReplicationTest and few fortifications(Paul Isaychuk via Pragya Mittal)
+
    FALCON-1698 New tests for ProcessSetupTest, ClusterSetupTest, UI test fixes(Paul Isaychuk
via Ajay Yadava)
 
    FALCON-1700 Add new test cases to HiveDRTest(Paul Isaychuk & Murali Ramasami via Ajay
Yadava)

http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java
index c1a7eb8..f1c408e 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java
@@ -58,6 +58,9 @@ public class Edge extends GraphEntity {
         @SerializedName("ninth")NINTH,
         @SerializedName("tenth")TENTH,
         @SerializedName("value")VALUE,
+        @SerializedName("_falcon_mirroring_type")MIRRORING_TYPE,
+        @SerializedName("specific")SPECIFIC,
+        @SerializedName("myTag1")MY_TAG
     }
     @SerializedName("_id")
     private String id;

http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java
index c9f540f..9d028fa 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java
@@ -18,7 +18,6 @@
 
 package org.apache.falcon.regression.core.util;
 
-import org.apache.commons.exec.CommandLine;
 import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants;
 import org.apache.hadoop.security.UserGroupInformation;
 
@@ -42,14 +41,6 @@ public final class KerberosHelper {
             getKeyTab(user));
     }
 
-    /**
-     * Switches user in kerberos.
-     */
-    public static void initUserWithKeytab(String user){
-        ExecUtil.executeCommand(new CommandLine("sudo").addArgument("kinit").addArgument(getPrincipal(user))
-            .addArgument("-k").addArgument("-t").addArgument(getKeyTab(user)));
-    }
-
     private static String getKeyTab(String user) {
         return MerlinConstants.getKeytabForUser(user);
     }

http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java
index ae96044..e73bc5d 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java
@@ -502,29 +502,45 @@ public final class OozieUtil {
         Assert.assertEquals(actualRun, attempts, "Rerun attempts did not match");
     }
 
+    /**
+     * Try to find feed coordinators of given type.
+     */
+    public static int checkIfFeedCoordExist(OozieClient oozieClient,
+                                            String feedName, String coordType) throws OozieClientException
{
+        return checkIfFeedCoordExist(oozieClient, feedName, coordType, 5);
+    }
+
+    /**
+     * Try to find feed coordinators of given type given number of times.
+     */
     public static int checkIfFeedCoordExist(OozieClient oozieClient,
-            String feedName, String coordType) throws OozieClientException {
+            String feedName, String coordType, int numberOfRetries) throws OozieClientException
{
         LOGGER.info("feedName: " + feedName);
-        int numberOfCoord = 0;
-        if (getBundles(oozieClient, feedName, EntityType.FEED).size() == 0) {
-            return 0;
-        }
-        List<String> bundleIds = getBundles(oozieClient, feedName, EntityType.FEED);
-        LOGGER.info("bundleIds: " + bundleIds);
-
-        for (String aBundleId : bundleIds) {
-            LOGGER.info("aBundleId: " + aBundleId);
-            waitForCoordinatorJobCreation(oozieClient, aBundleId);
-            List<CoordinatorJob> coords =
-                    getBundleCoordinators(oozieClient, aBundleId);
-            LOGGER.info("coords: " + coords);
-            for (CoordinatorJob coord : coords) {
-                if (coord.getAppName().contains(coordType)) {
-                    numberOfCoord++;
+        for (int retryAttempt = 0; retryAttempt < numberOfRetries; retryAttempt++) {
+            int numberOfCoord = 0;
+            List<String> bundleIds = getBundles(oozieClient, feedName, EntityType.FEED);
+            if (bundleIds.size() == 0) {
+                TimeUtil.sleepSeconds(4);
+                continue;
+            }
+            LOGGER.info("bundleIds: " + bundleIds);
+            for (String aBundleId : bundleIds) {
+                LOGGER.info("aBundleId: " + aBundleId);
+                waitForCoordinatorJobCreation(oozieClient, aBundleId);
+                List<CoordinatorJob> coords = getBundleCoordinators(oozieClient, aBundleId);
+                LOGGER.info("coords: " + coords);
+                for (CoordinatorJob coord : coords) {
+                    if (coord.getAppName().contains(coordType)) {
+                        numberOfCoord++;
+                    }
                 }
             }
+            if (numberOfCoord > 0) {
+                return numberOfCoord;
+            }
+            TimeUtil.sleepSeconds(4);
         }
-        return numberOfCoord;
+        return 0;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java
index e5430eb..60eeabd 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java
@@ -45,6 +45,7 @@ import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.impl.conn.BasicClientConnectionManager;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.util.EntityUtils;
 import org.apache.log4j.Logger;
 
 import javax.net.ssl.SSLContext;
@@ -198,6 +199,8 @@ public class BaseRequest {
                     LOGGER.info(String.format("Request Header: Name=%s Value=%s", header.getName(),
                         header.getValue()));
                 }
+                // Must call this to release the connection
+                EntityUtils.consume(response.getEntity());
                 response = client.execute(target, request);
             }
         }

http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java
index 1bffe9a..25cf16f 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java
@@ -105,6 +105,7 @@ public class TouchAPIPrismAndServerTest extends BaseTestClass {
 
         // via server
         oldbundleId = bundleId;
+        InstanceUtil.waitTillInstancesAreCreated(clusterOC, bundles[0].getProcessData(),
1);
         coordId = OozieUtil.getLatestCoordinatorID(clusterOC, bundles[0].getProcessName(),
EntityType.PROCESS);
         response = cluster.getProcessHelper().touchEntity(bundles[0].getProcessData());
         bundleId = OozieUtil.getLatestBundleID(clusterOC, bundles[0].getProcessName(), EntityType.PROCESS);

http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
index d639c21..49ed94a 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
@@ -262,7 +262,14 @@ public class HCatRetentionTest extends BaseTestClass {
             //Each HCat partition maps to a directory, not to a file
             HCatAddPartitionDesc addPtn = HCatAddPartitionDesc.create(dbName,
                 tableName, dataFolder, ptn).build();
-            client.addPartition(addPtn);
+            try {
+                client.addPartition(addPtn);
+            } catch (HCatException e) {
+                //occurrence of specific hive metastore bug
+                if (!e.getMessage().contains("AlreadyExistsException")) {
+                    throw e;
+                }
+            }
             ptn.clear();
         }
     }

http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java
index 17725ae..b3ca4f0 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java
@@ -69,8 +69,8 @@ public class ListFeedInstancesTest extends BaseTestClass {
     private String feedDataLocation = sourcePath + MINUTE_DATE_PATTERN;
     private String targetPath = baseTestHDFSDir + "/target";
     private String targetDataLocation = targetPath + MINUTE_DATE_PATTERN;
-    private final String startTime = "2010-01-02T00:00Z";
-    private final String endTime = "2010-01-02T00:57Z";
+    private final String startTime = "2015-01-02T00:00Z";
+    private final String endTime = "2015-01-02T00:57Z";
     private String feedName;
 
     @BeforeClass(alwaysRun = true)
@@ -210,7 +210,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
         throws URISyntaxException, OozieClientException, JAXBException, AuthenticationException,
         IOException, InterruptedException {
         SoftAssert softAssert = new SoftAssert();
-        //orderBy start time, check an order
+        //orderBy start time, check on order
         InstancesResult r = prism.getFeedHelper().listInstances(feedName,
             "orderBy=startTime&sortOrder=desc", null);
         InstancesResult.Instance[] instances = r.getInstances();
@@ -223,7 +223,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
                 previousDate = (Date) current.clone();
             }
         }
-        //orderBy status, check an order
+        //orderBy status, check on order
         r = prism.getFeedHelper().listInstances(feedName,
             "start=" + startTime + "&numResults=12&orderBy=status&sortOrder=desc",
null);
         InstanceUtil.validateResponse(r, 12, 1, 1, 8, 2);
@@ -235,7 +235,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
                 "Wrong order. Compared " + current + " and " + previousStatus + " statuses.");
             previousStatus = current;
         }
-        //sort by endTime, check an order
+        //sort by endTime, check on order
         r = prism.getFeedHelper().listInstances(feedName,
             "start=" + startTime + "&numResults=12&orderBy=endTime&sortOrder=desc",
null);
         instances = r.getInstances();
@@ -252,7 +252,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
     }
 
     /**
-     * List instances through api using start/end parameters. Check a number of instances.
+     * Test the list feed instance api using start/end parameters. Check instances number.
      */
     @Test
     public void testFeedStartEnd()

http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
index 8f45d1c..018b83a 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
@@ -54,6 +54,7 @@ import javax.jms.JMSException;
 import javax.jms.MapMessage;
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -181,10 +182,13 @@ public class RetentionTest extends BaseTestClass {
         JmsMessageConsumer messageConsumer = new JmsMessageConsumer("FALCON." + feedName,
                 cluster.getClusterHelper().getActiveMQ());
         messageConsumer.start();
-        final DateTime currentTime = new DateTime(DateTimeZone.UTC);
         String bundleId = OozieUtil.getBundles(clusterOC, feedName, EntityType.FEED).get(0);
 
         List<String> workflows = OozieUtil.waitForRetentionWorkflowToSucceed(bundleId,
clusterOC);
+
+        //get current time minus duration of last status check - to get actual time when
eviction has started
+        final DateTime currentTime = new DateTime(new DateTime(DateTimeZone.UTC).toDate().getTime()
- 10000);
+        LOGGER.info("Current time is " + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(currentTime.toDate()));
         LOGGER.info("workflows: " + workflows);
         messageConsumer.interrupt();
         Util.printMessageData(messageConsumer);

http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java
index 73273f9..d11411b 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java
@@ -25,7 +25,6 @@ import org.apache.falcon.regression.core.supportClasses.ExecResult;
 import org.apache.falcon.regression.core.util.AssertUtil;
 import org.apache.falcon.regression.core.util.BundleUtil;
 import org.apache.falcon.regression.core.util.HadoopUtil;
-import org.apache.falcon.regression.core.util.KerberosHelper;
 import org.apache.falcon.regression.core.util.OSUtil;
 import org.apache.falcon.regression.testHelper.BaseTestClass;
 import org.apache.hadoop.fs.FileSystem;
@@ -81,13 +80,9 @@ public class FalconClientTest extends BaseTestClass {
      * able to delete
      * @throws Exception
      */
-    @Test(enabled = true)
+    @Test(enabled = false)
     public void badClusterDelete() throws Exception {
         bundles[0].submitClusters(prism);
-        //switch user
-        if (MerlinConstants.IS_SECURE) {
-            KerberosHelper.initUserWithKeytab(MerlinConstants.DIFFERENT_USER_NAME);
-        }
         final String clusterXml = bundles[0].getClusters().get(0);
         final ExecResult execResult =
             prism.getClusterHelper().clientDelete(clusterXml, MerlinConstants.DIFFERENT_USER_NAME);


Mime
View raw message