falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ar...@apache.org
Subject [03/18] git commit: FALCON-581 merlin: Refactor code for cross product and make it a method. Contributed by Raghav Kumar Gautam
Date Thu, 21 Aug 2014 17:30:45 GMT
FALCON-581 merlin: Refactor code for cross product and make it a method. Contributed by Raghav
Kumar Gautam


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/d6b5f417
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/d6b5f417
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/d6b5f417

Branch: refs/heads/FALCON-585
Commit: d6b5f417ad84740b3370e26e69cb4a70fd9c1e25
Parents: aee323d
Author: arpit <arpit@apache.org>
Authored: Thu Aug 14 08:13:51 2014 -0700
Committer: arpit <arpit@apache.org>
Committed: Thu Aug 14 08:13:51 2014 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 +
 .../falcon/regression/core/util/MathUtil.java   | 60 ++++++++++++++++++++
 .../apache/falcon/regression/NewRetryTest.java  | 26 ++++-----
 .../regression/hcat/HCatRetentionTest.java      | 19 +------
 .../regression/prism/PrismSubmitTest.java       | 10 ----
 .../falcon/regression/prism/RetentionTest.java  | 27 ++-------
 6 files changed, 81 insertions(+), 64 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e2c5df3..085773b 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -21,6 +21,9 @@ Trunk (Unreleased)
    FALCON-263 API to get workflow parameters. (pavan kumar kolamuri via Shwetha GS)
 
   IMPROVEMENTS
+   FALCON-581 merlin: Refactor code for cross product and make it a method 
+   (Raghav Kumar Gautam via Arpit Gupta) 
+   
    FALCON-597 String logged at the start and end of the merlin test are slightly 
    mismatched (Raghav Kumar Gautam via Arpit Gupta)
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java
new file mode 100644
index 0000000..1bbe8d9
--- /dev/null
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.regression.core.util;
+
+public class MathUtil {
+    private MathUtil() {
+        throw new AssertionError("Instantiating utility class...");
+    }
+
+    /**
+     * Cross product many arrays
+     * @param firstArray first array that you want to cross product
+     * @param otherArrays other arrays that you want to cross product
+     * @return cross product
+     */
+    public static Object[][] crossProduct(Object[] firstArray, Object[]... otherArrays) {
+        if(otherArrays == null || otherArrays.length == 0) {
+            Object[][] result = new Object[firstArray.length][1];
+            for (int i = 0; i < firstArray.length; ++i) {
+                result[i][0] = firstArray[i];
+            }
+            return result;
+        }
+        // computing cross product for the rest of the arrays
+        Object[][] restArray = new Object[otherArrays.length-1][];
+        System.arraycopy(otherArrays, 1, restArray, 0, otherArrays.length - 1);
+        Object[][] restCrossProduct = crossProduct(otherArrays[0], restArray);
+        //creating and initializing result array
+        Object[][] result = new Object[firstArray.length * restCrossProduct.length][];
+        for(int i = 0; i < result.length; ++i) {
+            result[i] = new Object[otherArrays.length + 1];
+        }
+        //doing the final cross product
+        for (int i = 0; i < firstArray.length; ++i) {
+            for (int j = 0; j < restCrossProduct.length; ++j) {
+                //computing one row of result
+                final int rowIdx = i * restCrossProduct.length + j;
+                result[rowIdx][0] = firstArray[i];
+                System.arraycopy(restCrossProduct[j], 0, result[rowIdx], 1, otherArrays.length);
+            }
+        }
+        return result;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java
index 65cf14e..5ab3dfe 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java
@@ -31,6 +31,7 @@ import org.apache.falcon.regression.core.util.AssertUtil;
 import org.apache.falcon.regression.core.util.BundleUtil;
 import org.apache.falcon.regression.core.util.HadoopUtil;
 import org.apache.falcon.regression.core.util.InstanceUtil;
+import org.apache.falcon.regression.core.util.MathUtil;
 import org.apache.falcon.regression.core.util.OSUtil;
 import org.apache.falcon.regression.core.util.OozieUtil;
 import org.apache.falcon.regression.core.util.TimeUtil;
@@ -996,27 +997,20 @@ public class NewRetryTest extends BaseTestClass {
     public Object[][] getData() {
 
         String[] retryTypes = new String[]{"periodic", "exp-backoff"};//,"exp-backoff"
-        int[] delays = new int[]{2,
+        Integer[] delays = new Integer[]{2,
             0};//removing -1 since this should be checked at validation level while setting
         String[] delayUnits = new String[]{"minutes"};
         Integer[] retryAttempts = new Integer[]{2, 0, 3};//0,-1,2
 
-        Object[][] testData = new Object[retryTypes.length * delays.length * delayUnits.length
*
-            retryAttempts.length][1];
-
-        int i = 0;
-
-        for (String retryType : retryTypes) {
-            for (int delay : delays) {
-                for (String delayUnit : delayUnits) {
-                    for (int retry : retryAttempts) {
-                        testData[i][0] = getRetry(delay, delayUnit, retryType, retry);
-                        i++;
-                    }
-                }
-            }
+        Object[][] crossProd = MathUtil.crossProduct(delays, delayUnits, retryTypes, retryAttempts);
+        Object[][] testData = new Object[crossProd.length][1];
+        for (int i = 0; i < crossProd.length; ++i) {
+            final Integer delay = (Integer) crossProd[i][0];
+            final String delayUnit = (String) crossProd[i][1];
+            final String retryType = (String) crossProd[i][2];
+            final Integer retry = (Integer) crossProd[i][3];
+            testData[i][0] = getRetry(delay, delayUnit, retryType, retry);
         }
-
         return testData;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
index 1d5fb1d..cce2860 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
@@ -27,6 +27,7 @@ import org.apache.falcon.regression.core.helpers.ColoHelper;
 import org.apache.falcon.regression.core.util.AssertUtil;
 import org.apache.falcon.regression.core.util.BundleUtil;
 import org.apache.falcon.regression.core.util.HCatUtil;
+import org.apache.falcon.regression.core.util.MathUtil;
 import org.apache.falcon.regression.core.util.OSUtil;
 import org.apache.falcon.regression.core.util.OozieUtil;
 import org.apache.falcon.regression.core.util.HadoopUtil;
@@ -276,28 +277,14 @@ public class HCatRetentionTest extends BaseTestClass {
     public Object[][] getTestData(Method m) {
         RetentionUnit[] retentionUnits = new RetentionUnit[]{RetentionUnit.HOURS, RetentionUnit.DAYS,
             RetentionUnit.MONTHS};// "minutes","years",
-        int[] periods = new int[]{7, 824, 43}; // a negative value like -4 should be covered
+        Integer[] periods = new Integer[]{7, 824, 43}; // a negative value like -4 should
be covered
         // in validation scenarios.
         FeedType[] dataTypes =
             new FeedType[]{
                 //disabling since falcon has support is for only for single hcat partition
                 //FeedType.DAILY, FeedType.MINUTELY, FeedType.HOURLY, FeedType.MONTHLY,
                 FeedType.YEARLY};
-        Object[][] testData = new Object[retentionUnits.length * periods.length * dataTypes.length][3];
-
-        int i = 0;
-
-        for (RetentionUnit retentionUnit : retentionUnits) {
-            for (int period : periods) {
-                for (FeedType dataType : dataTypes) {
-                    testData[i][0] = period;
-                    testData[i][1] = retentionUnit;
-                    testData[i][2] = dataType;
-                    i++;
-                }
-            }
-        }
-        return testData;
+        return MathUtil.crossProduct(periods, retentionUnits, dataTypes);
     }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java
index 8156937..8f2b160 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java
@@ -33,7 +33,6 @@ import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 import java.lang.reflect.Method;
@@ -590,13 +589,4 @@ public class PrismSubmitTest extends BaseTestClass {
         AssertUtil.compareDataStoreStates(beforeSubmitCluster2, afterSubmitCluster2, 0);
     }
 
-    @DataProvider(name = "errorDP")
-    public Object[][] getTestData(Method m) {
-        Object[][] testData = new Object[2][1];
-        testData[0][0] = "EmptyInputTagProcess";
-        testData[1][0] = "EmptyOutputTagProcess";
-
-        return testData;
-    }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
index c8cef02..4937361 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
@@ -30,6 +30,7 @@ import org.apache.falcon.regression.core.supportClasses.JmsMessageConsumer;
 import org.apache.falcon.regression.core.util.AssertUtil;
 import org.apache.falcon.regression.core.util.BundleUtil;
 import org.apache.falcon.regression.core.util.HadoopUtil;
+import org.apache.falcon.regression.core.util.MathUtil;
 import org.apache.falcon.regression.core.util.OozieUtil;
 import org.apache.falcon.regression.core.util.TimeUtil;
 import org.apache.falcon.regression.core.util.Util;
@@ -226,32 +227,14 @@ public class RetentionTest extends BaseTestClass {
     @DataProvider(name = "betterDP")
     public Object[][] getTestData(Method m) {
         // a negative value like -4 should be covered in validation scenarios.
-        int[] retentionPeriods = new int[]{0, 10080, 60, 8, 24};
+        Integer[] retentionPeriods = new Integer[]{0, 10080, 60, 8, 24};
         RetentionUnit[] retentionUnits = new RetentionUnit[]{RetentionUnit.HOURS,
             RetentionUnit.DAYS};// "minutes","hours", "days",
-        boolean[] gaps = new boolean[]{false, true};
+        Boolean[] gaps = new Boolean[]{false, true};
         FeedType[] feedTypes = new FeedType[]{FeedType.DAILY, FeedType.YEARLY, FeedType.MONTHLY};
-        Object[][] testData = new Object[retentionPeriods.length * retentionUnits.length
*
-            gaps.length * feedTypes.length][5];
-
-        int i = 0;
-
-        for (RetentionUnit retentionUnit : retentionUnits) {
-            for (int period : retentionPeriods) {
-                for (boolean gap : gaps) {
-                    for (FeedType feedType : feedTypes) {
-                        testData[i][0] = period;
-                        testData[i][1] = retentionUnit;
-                        testData[i][2] = gap;
-                        testData[i][3] = feedType;
-                        testData[i][4] = true;
-                        i++;
-                    }
-                }
-            }
-        }
+        final Boolean[] withData = new Boolean[]{true};
 
-        return testData;
+        return MathUtil.crossProduct(retentionPeriods, retentionUnits, gaps, feedTypes, withData);
     }
 
 }


Mime
View raw message