Return-Path: X-Original-To: apmail-pig-commits-archive@www.apache.org Delivered-To: apmail-pig-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 775C718DFD for ; Wed, 16 Mar 2016 13:23:49 +0000 (UTC) Received: (qmail 95165 invoked by uid 500); 16 Mar 2016 13:23:49 -0000 Delivered-To: apmail-pig-commits-archive@pig.apache.org Received: (qmail 93873 invoked by uid 500); 16 Mar 2016 13:23:34 -0000 Mailing-List: contact commits-help@pig.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@pig.apache.org Delivered-To: mailing list commits@pig.apache.org Received: (qmail 93805 invoked by uid 99); 16 Mar 2016 13:23:34 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Mar 2016 13:23:34 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id DFB75C2CE9 for ; Wed, 16 Mar 2016 13:23:33 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.799 X-Spam-Level: * X-Spam-Status: No, score=1.799 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RP_MATCHES_RCVD=-0.001] autolearn=disabled Received: from mx2-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id gB8q2IwfTVq8 for ; Wed, 16 Mar 2016 13:23:32 +0000 (UTC) Received: from mailrelay1-us-west.apache.org (mailrelay1-us-west.apache.org [209.188.14.139]) by mx2-lw-eu.apache.org (ASF Mail Server at mx2-lw-eu.apache.org) with ESMTP id 165F05F56C for ; Wed, 16 Mar 2016 13:23:32 +0000 (UTC) Received: from svn01-us-west.apache.org (svn.apache.org [10.41.0.6]) by mailrelay1-us-west.apache.org (ASF Mail Server at mailrelay1-us-west.apache.org) with ESMTP id 030D1E0182 for ; Wed, 16 Mar 2016 13:23:30 +0000 (UTC) Received: from svn01-us-west.apache.org (localhost [127.0.0.1]) by svn01-us-west.apache.org (ASF Mail Server at svn01-us-west.apache.org) with ESMTP id D5EED3A0734 for ; Wed, 16 Mar 2016 13:23:30 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1735229 - in /pig/branches/spark: src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java test/org/apache/pig/test/TestBuiltin.java Date: Wed, 16 Mar 2016 13:23:30 -0000 To: commits@pig.apache.org From: xuefu@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20160316132330.D5EED3A0734@svn01-us-west.apache.org> Author: xuefu Date: Wed Mar 16 13:23:30 2016 New Revision: 1735229 URL: http://svn.apache.org/viewvc?rev=1735229&view=rev Log: PIG-4838: Fix test TestBuiltin (Liyun via Xuefu) Modified: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java pig/branches/spark/test/org/apache/pig/test/TestBuiltin.java Modified: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java?rev=1735229&r1=1735228&r2=1735229&view=diff ============================================================================== --- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java (original) +++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java Wed Mar 16 13:23:30 2016 @@ -21,9 +21,12 @@ import java.io.IOException; import java.io.Serializable; import java.util.Iterator; import java.util.List; +import java.util.UUID; import org.apache.hadoop.mapred.JobConf; +import org.apache.pig.PigConstants; import org.apache.pig.backend.executionengine.ExecException; +import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRConfiguration; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.ProgressableReporter; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; @@ -74,18 +77,20 @@ public class ForEachConverter implements } void initializeJobConf() { - if (this.jobConf == null) { - this.jobConf = KryoSerializer.deserializeJobConf(this.confBytes); - PigMapReduce.sJobConfInternal.set(jobConf); - try { - MapRedUtil.setupUDFContext(jobConf); - PigContext pc = (PigContext) ObjectSerializer.deserialize(jobConf.get("pig.pigContext")); - SchemaTupleBackend.initialize(jobConf, pc); - - } catch (IOException ioe) { - String msg = "Problem while configuring UDFContext from ForEachConverter."; - throw new RuntimeException(msg, ioe); - } + if (this.jobConf != null) { + return; + } + this.jobConf = KryoSerializer.deserializeJobConf(this.confBytes); + PigMapReduce.sJobConfInternal.set(jobConf); + try { + MapRedUtil.setupUDFContext(jobConf); + PigContext pc = (PigContext) ObjectSerializer.deserialize(jobConf.get("pig.pigContext")); + SchemaTupleBackend.initialize(jobConf, pc); + jobConf.set(MRConfiguration.JOB_ID, UUID.randomUUID().toString()); + jobConf.set(PigConstants.TASK_INDEX, "0"); + } catch (IOException ioe) { + String msg = "Problem while configuring UDFContext from ForEachConverter."; + throw new RuntimeException(msg, ioe); } } Modified: pig/branches/spark/test/org/apache/pig/test/TestBuiltin.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestBuiltin.java?rev=1735229&r1=1735228&r2=1735229&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestBuiltin.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestBuiltin.java Wed Mar 16 13:23:30 2016 @@ -3215,16 +3215,46 @@ public class TestBuiltin { pigServer.registerQuery("A = load '" + inputFileName + "' as (name);"); pigServer.registerQuery("B = foreach A generate name, UniqueID();"); Iterator iter = pigServer.openIterator("B"); - assertEquals(iter.next().get(1),"0-0"); - assertEquals(iter.next().get(1),"0-1"); - assertEquals(iter.next().get(1),"0-2"); - assertEquals(iter.next().get(1),"0-3"); - assertEquals(iter.next().get(1),"0-4"); - assertEquals(iter.next().get(1),"1-0"); - assertEquals(iter.next().get(1),"1-1"); - assertEquals(iter.next().get(1),"1-2"); - assertEquals(iter.next().get(1),"1-3"); - assertEquals(iter.next().get(1),"1-4"); + if (!Util.isSparkExecType(cluster.getExecType())) { + assertEquals(iter.next().get(1), "0-0"); + assertEquals(iter.next().get(1), "0-1"); + assertEquals(iter.next().get(1), "0-2"); + assertEquals(iter.next().get(1), "0-3"); + assertEquals(iter.next().get(1), "0-4"); + assertEquals(iter.next().get(1), "1-0"); + assertEquals(iter.next().get(1), "1-1"); + assertEquals(iter.next().get(1), "1-2"); + assertEquals(iter.next().get(1), "1-3"); + assertEquals(iter.next().get(1), "1-4"); + } else{ + //because we set PigConstants.TASK_INDEX as 0 in ForEachConverter#ForEachFunction#initializeJobConf + //UniqueID.exec() will output like 0-* + //there will be 2 InputSplits when mapred.max.split.size is 10(byte) for the testUniqueID.txt(20 bytes) + //Split0: + // 1\n + // 2\n + // 3\n + // 4\n + // 5\n + // 1\n + //Split1: + // 2\n + // 3\n + // 4\n + // 5\n + //The size of Split0 is 12 not 10 because LineRecordReader#nextKeyValue will read one more line + //More detail see PIG-4383 + assertEquals(iter.next().get(1), "0-0"); + assertEquals(iter.next().get(1), "0-1"); + assertEquals(iter.next().get(1), "0-2"); + assertEquals(iter.next().get(1), "0-3"); + assertEquals(iter.next().get(1), "0-4"); + assertEquals(iter.next().get(1), "0-5"); + assertEquals(iter.next().get(1), "0-0"); + assertEquals(iter.next().get(1), "0-1"); + assertEquals(iter.next().get(1), "0-2"); + assertEquals(iter.next().get(1), "0-3"); + } } @Test