Author: rohini
Date: Fri Mar 14 20:24:01 2014
New Revision: 1577691
URL: http://svn.apache.org/r1577691
Log:
PIG-3810: Implement CROSS in Tez
Modified:
pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java
pig/branches/tez/test/org/apache/pig/test/TestCustomPartitioner.java
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java?rev=1577691&r1=1577690&r2=1577691&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java
(original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java
Fri Mar 14 20:24:01 2014
@@ -141,7 +141,7 @@ public class TezCompiler extends PhyPlan
private UDFFinder udfFinder;
private Map<PhysicalOperator, TezOperator> phyToTezOpMap;
-
+
private TezResourceManager tezResourceManager;
public static final String USER_COMPARATOR_MARKER = "user.comparator.func:";
@@ -159,7 +159,7 @@ public class TezCompiler extends PhyPlan
this.plan = plan;
this.pigContext = pigContext;
this.tezResourceManager = tezResourceManager;
-
+
pigProperties = pigContext.getProperties();
splitsSeen = Maps.newHashMap();
tezPlan = new TezOperPlan(tezResourceManager);
@@ -572,9 +572,14 @@ public class TezCompiler extends PhyPlan
@Override
public void visitCross(POCross op) throws VisitorException {
- int errCode = 2034;
- String msg = "Cannot compile " + op.getClass().getSimpleName();
- throw new TezCompilerException(msg, errCode, PigException.BUG);
+ try{
+ nonBlocking(op);
+ phyToTezOpMap.put(op, curTezOp);
+ } catch (Exception e) {
+ int errCode = 2034;
+ String msg = "Error compiling operator " + op.getClass().getSimpleName();
+ throw new TezCompilerException(msg, errCode, PigException.BUG, e);
+ }
}
@Override
Modified: pig/branches/tez/test/org/apache/pig/test/TestCustomPartitioner.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestCustomPartitioner.java?rev=1577691&r1=1577690&r2=1577691&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestCustomPartitioner.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestCustomPartitioner.java Fri Mar 14 20:24:01
2014
@@ -42,7 +42,6 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
public class TestCustomPartitioner {
@@ -74,8 +73,6 @@ public class TestCustomPartitioner {
// See PIG-282
@Test
- @Ignore
- // Fails with Tez - DefaultSorter.java Illegal partition for Null: false index: 0 1 (-1),
TotalPartitions: 0
public void testCustomPartitionerParseJoins() throws Exception{
String[] input = {
"1\t3",
@@ -212,8 +209,6 @@ public class TestCustomPartitioner {
// See PIG-282
@Test
- @Ignore
- // TODO: CROSS not implemented in TEZ yet
public void testCustomPartitionerCross() throws Exception{
String[] input = {
"1\t3",
|