Return-Path: X-Original-To: apmail-falcon-commits-archive@minotaur.apache.org Delivered-To: apmail-falcon-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 45E5E11F93 for ; Thu, 14 Aug 2014 15:14:23 +0000 (UTC) Received: (qmail 31081 invoked by uid 500); 14 Aug 2014 15:14:23 -0000 Delivered-To: apmail-falcon-commits-archive@falcon.apache.org Received: (qmail 31046 invoked by uid 500); 14 Aug 2014 15:14:23 -0000 Mailing-List: contact commits-help@falcon.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@falcon.incubator.apache.org Delivered-To: mailing list commits@falcon.incubator.apache.org Received: (qmail 31030 invoked by uid 99); 14 Aug 2014 15:14:22 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 14 Aug 2014 15:14:22 +0000 X-ASF-Spam-Status: No, hits=-2000.7 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Thu, 14 Aug 2014 15:14:20 +0000 Received: (qmail 30311 invoked by uid 99); 14 Aug 2014 15:14:00 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 14 Aug 2014 15:14:00 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 1E81C9C3087; Thu, 14 Aug 2014 15:14:00 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: arpit@apache.org To: commits@falcon.incubator.apache.org Message-Id: <609cb002b43249eca6fff37f72c01950@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: FALCON-581 merlin: Refactor code for cross product and make it a method. Contributed by Raghav Kumar Gautam Date: Thu, 14 Aug 2014 15:14:00 +0000 (UTC) X-Virus-Checked: Checked by ClamAV on apache.org Repository: incubator-falcon Updated Branches: refs/heads/master aee323d4c -> d6b5f417a FALCON-581 merlin: Refactor code for cross product and make it a method. Contributed by Raghav Kumar Gautam Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/d6b5f417 Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/d6b5f417 Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/d6b5f417 Branch: refs/heads/master Commit: d6b5f417ad84740b3370e26e69cb4a70fd9c1e25 Parents: aee323d Author: arpit Authored: Thu Aug 14 08:13:51 2014 -0700 Committer: arpit Committed: Thu Aug 14 08:13:51 2014 -0700 ---------------------------------------------------------------------- CHANGES.txt | 3 + .../falcon/regression/core/util/MathUtil.java | 60 ++++++++++++++++++++ .../apache/falcon/regression/NewRetryTest.java | 26 ++++----- .../regression/hcat/HCatRetentionTest.java | 19 +------ .../regression/prism/PrismSubmitTest.java | 10 ---- .../falcon/regression/prism/RetentionTest.java | 27 ++------- 6 files changed, 81 insertions(+), 64 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index e2c5df3..085773b 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -21,6 +21,9 @@ Trunk (Unreleased) FALCON-263 API to get workflow parameters. (pavan kumar kolamuri via Shwetha GS) IMPROVEMENTS + FALCON-581 merlin: Refactor code for cross product and make it a method + (Raghav Kumar Gautam via Arpit Gupta) + FALCON-597 String logged at the start and end of the merlin test are slightly mismatched (Raghav Kumar Gautam via Arpit Gupta) http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java new file mode 100644 index 0000000..1bbe8d9 --- /dev/null +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.regression.core.util; + +public class MathUtil { + private MathUtil() { + throw new AssertionError("Instantiating utility class..."); + } + + /** + * Cross product many arrays + * @param firstArray first array that you want to cross product + * @param otherArrays other arrays that you want to cross product + * @return cross product + */ + public static Object[][] crossProduct(Object[] firstArray, Object[]... otherArrays) { + if(otherArrays == null || otherArrays.length == 0) { + Object[][] result = new Object[firstArray.length][1]; + for (int i = 0; i < firstArray.length; ++i) { + result[i][0] = firstArray[i]; + } + return result; + } + // computing cross product for the rest of the arrays + Object[][] restArray = new Object[otherArrays.length-1][]; + System.arraycopy(otherArrays, 1, restArray, 0, otherArrays.length - 1); + Object[][] restCrossProduct = crossProduct(otherArrays[0], restArray); + //creating and initializing result array + Object[][] result = new Object[firstArray.length * restCrossProduct.length][]; + for(int i = 0; i < result.length; ++i) { + result[i] = new Object[otherArrays.length + 1]; + } + //doing the final cross product + for (int i = 0; i < firstArray.length; ++i) { + for (int j = 0; j < restCrossProduct.length; ++j) { + //computing one row of result + final int rowIdx = i * restCrossProduct.length + j; + result[rowIdx][0] = firstArray[i]; + System.arraycopy(restCrossProduct[j], 0, result[rowIdx], 1, otherArrays.length); + } + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java index 65cf14e..5ab3dfe 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java @@ -31,6 +31,7 @@ import org.apache.falcon.regression.core.util.AssertUtil; import org.apache.falcon.regression.core.util.BundleUtil; import org.apache.falcon.regression.core.util.HadoopUtil; import org.apache.falcon.regression.core.util.InstanceUtil; +import org.apache.falcon.regression.core.util.MathUtil; import org.apache.falcon.regression.core.util.OSUtil; import org.apache.falcon.regression.core.util.OozieUtil; import org.apache.falcon.regression.core.util.TimeUtil; @@ -996,27 +997,20 @@ public class NewRetryTest extends BaseTestClass { public Object[][] getData() { String[] retryTypes = new String[]{"periodic", "exp-backoff"};//,"exp-backoff" - int[] delays = new int[]{2, + Integer[] delays = new Integer[]{2, 0};//removing -1 since this should be checked at validation level while setting String[] delayUnits = new String[]{"minutes"}; Integer[] retryAttempts = new Integer[]{2, 0, 3};//0,-1,2 - Object[][] testData = new Object[retryTypes.length * delays.length * delayUnits.length * - retryAttempts.length][1]; - - int i = 0; - - for (String retryType : retryTypes) { - for (int delay : delays) { - for (String delayUnit : delayUnits) { - for (int retry : retryAttempts) { - testData[i][0] = getRetry(delay, delayUnit, retryType, retry); - i++; - } - } - } + Object[][] crossProd = MathUtil.crossProduct(delays, delayUnits, retryTypes, retryAttempts); + Object[][] testData = new Object[crossProd.length][1]; + for (int i = 0; i < crossProd.length; ++i) { + final Integer delay = (Integer) crossProd[i][0]; + final String delayUnit = (String) crossProd[i][1]; + final String retryType = (String) crossProd[i][2]; + final Integer retry = (Integer) crossProd[i][3]; + testData[i][0] = getRetry(delay, delayUnit, retryType, retry); } - return testData; } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java index 1d5fb1d..cce2860 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java @@ -27,6 +27,7 @@ import org.apache.falcon.regression.core.helpers.ColoHelper; import org.apache.falcon.regression.core.util.AssertUtil; import org.apache.falcon.regression.core.util.BundleUtil; import org.apache.falcon.regression.core.util.HCatUtil; +import org.apache.falcon.regression.core.util.MathUtil; import org.apache.falcon.regression.core.util.OSUtil; import org.apache.falcon.regression.core.util.OozieUtil; import org.apache.falcon.regression.core.util.HadoopUtil; @@ -276,28 +277,14 @@ public class HCatRetentionTest extends BaseTestClass { public Object[][] getTestData(Method m) { RetentionUnit[] retentionUnits = new RetentionUnit[]{RetentionUnit.HOURS, RetentionUnit.DAYS, RetentionUnit.MONTHS};// "minutes","years", - int[] periods = new int[]{7, 824, 43}; // a negative value like -4 should be covered + Integer[] periods = new Integer[]{7, 824, 43}; // a negative value like -4 should be covered // in validation scenarios. FeedType[] dataTypes = new FeedType[]{ //disabling since falcon has support is for only for single hcat partition //FeedType.DAILY, FeedType.MINUTELY, FeedType.HOURLY, FeedType.MONTHLY, FeedType.YEARLY}; - Object[][] testData = new Object[retentionUnits.length * periods.length * dataTypes.length][3]; - - int i = 0; - - for (RetentionUnit retentionUnit : retentionUnits) { - for (int period : periods) { - for (FeedType dataType : dataTypes) { - testData[i][0] = period; - testData[i][1] = retentionUnit; - testData[i][2] = dataType; - i++; - } - } - } - return testData; + return MathUtil.crossProduct(periods, retentionUnits, dataTypes); } } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java index 8156937..8f2b160 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java @@ -33,7 +33,6 @@ import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.lang.reflect.Method; @@ -590,13 +589,4 @@ public class PrismSubmitTest extends BaseTestClass { AssertUtil.compareDataStoreStates(beforeSubmitCluster2, afterSubmitCluster2, 0); } - @DataProvider(name = "errorDP") - public Object[][] getTestData(Method m) { - Object[][] testData = new Object[2][1]; - testData[0][0] = "EmptyInputTagProcess"; - testData[1][0] = "EmptyOutputTagProcess"; - - return testData; - } - } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java index c8cef02..4937361 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java @@ -30,6 +30,7 @@ import org.apache.falcon.regression.core.supportClasses.JmsMessageConsumer; import org.apache.falcon.regression.core.util.AssertUtil; import org.apache.falcon.regression.core.util.BundleUtil; import org.apache.falcon.regression.core.util.HadoopUtil; +import org.apache.falcon.regression.core.util.MathUtil; import org.apache.falcon.regression.core.util.OozieUtil; import org.apache.falcon.regression.core.util.TimeUtil; import org.apache.falcon.regression.core.util.Util; @@ -226,32 +227,14 @@ public class RetentionTest extends BaseTestClass { @DataProvider(name = "betterDP") public Object[][] getTestData(Method m) { // a negative value like -4 should be covered in validation scenarios. - int[] retentionPeriods = new int[]{0, 10080, 60, 8, 24}; + Integer[] retentionPeriods = new Integer[]{0, 10080, 60, 8, 24}; RetentionUnit[] retentionUnits = new RetentionUnit[]{RetentionUnit.HOURS, RetentionUnit.DAYS};// "minutes","hours", "days", - boolean[] gaps = new boolean[]{false, true}; + Boolean[] gaps = new Boolean[]{false, true}; FeedType[] feedTypes = new FeedType[]{FeedType.DAILY, FeedType.YEARLY, FeedType.MONTHLY}; - Object[][] testData = new Object[retentionPeriods.length * retentionUnits.length * - gaps.length * feedTypes.length][5]; - - int i = 0; - - for (RetentionUnit retentionUnit : retentionUnits) { - for (int period : retentionPeriods) { - for (boolean gap : gaps) { - for (FeedType feedType : feedTypes) { - testData[i][0] = period; - testData[i][1] = retentionUnit; - testData[i][2] = gap; - testData[i][3] = feedType; - testData[i][4] = true; - i++; - } - } - } - } + final Boolean[] withData = new Boolean[]{true}; - return testData; + return MathUtil.crossProduct(retentionPeriods, retentionUnits, gaps, feedTypes, withData); } }