pig-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Pallavi Rao <pallavi....@inmobi.com>
Subject Re: Review Request 45667: Support Pig On Spark
Date Thu, 27 Oct 2016 03:18:08 GMT

-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/45667/
-----------------------------------------------------------

(Updated Oct. 27, 2016, 3:17 a.m.)


Review request for pig, Daniel Dai and Rohini Palaniswamy.


Bugs: PIG-4059 and PIG-4854
    https://issues.apache.org/jira/browse/PIG-4059
    https://issues.apache.org/jira/browse/PIG-4854


Repository: pig-git


Description
-------

The patch contains all the work done in the spark branch, so far.


Diffs (updated)
-----

  bin/pig 81f1426 
  build.xml 99ba1f4 
  ivy.xml dd9878e 
  ivy/libraries.properties 3a819a5 
  shims/test/hadoop20/org/apache/pig/test/SparkMiniCluster.java PRE-CREATION 
  shims/test/hadoop23/org/apache/pig/test/SparkMiniCluster.java PRE-CREATION 
  shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java 792a1bd 
  shims/test/hadoop23/org/apache/pig/test/YarnMiniCluster.java PRE-CREATION 
  src/META-INF/services/org.apache.pig.ExecType 5c034c8 
  src/docs/src/documentation/content/xdocs/start.xml 36f9952 
  src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java 1ff1abd

  src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
ecf780c 
  src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhysicalPlan.java
2376d03 
  src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCollectedGroup.java
bcbfe2b 
  src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java
d80951a 
  src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java
21b75f1 
  src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POGlobalRearrange.java
52cfb73 
  src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeJoin.java
13f70c0 
  src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java
c3a82c3 
  src/org/apache/pig/backend/hadoop/executionengine/spark/JobGraphBuilder.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/JobMetricsListener.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/KryoSerializer.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/MapReducePartitionerWrapper.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/SparkEngineConf.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecType.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecutionEngine.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLocalExecType.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/SparkUtil.java PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/UDFJarsFinder.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/CollectedGroupConverter.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/CounterConverter.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/DistinctConverter.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/FRJoinConverter.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/FilterConverter.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/GlobalRearrangeConverter.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IndexedKey.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LimitConverter.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LoadConverter.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LocalRearrangeConverter.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/MergeCogroupConverter.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/MergeJoinConverter.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/OutputConsumerIterator.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/PackageConverter.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/PigSecondaryKeyComparatorSpark.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/RDDConverter.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/RankConverter.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ReduceByConverter.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SortConverter.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SplitConverter.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/StoreConverter.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/StreamConverter.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/converter/UnionConverter.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/operator/NativeSparkOperator.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/operator/POGlobalRearrangeSpark.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/operator/POReduceBySpark.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/optimizer/AccumulatorOptimizer.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/optimizer/CombinerOptimizer.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/optimizer/MultiQueryOptimizerSpark.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/optimizer/NoopFilterRemover.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/optimizer/ParallelismSetter.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/optimizer/SecondaryKeyOptimizerSpark.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/plan/DotSparkPrinter.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/plan/SparkCompiler.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/plan/SparkCompilerException.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/plan/SparkOpPlanVisitor.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/plan/SparkOperPlan.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/plan/SparkOperator.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/plan/SparkPOPackageAnnotator.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/spark/plan/SparkPrinter.java PRE-CREATION

  src/org/apache/pig/backend/hadoop/executionengine/spark/running/PigInputFormatSpark.java
PRE-CREATION 
  src/org/apache/pig/backend/hadoop/executionengine/util/AccumulatorOptimizerUtil.java c4b44ad

  src/org/apache/pig/backend/hadoop/executionengine/util/CombinerOptimizerUtil.java 889c01b

  src/org/apache/pig/backend/hadoop/executionengine/util/SecondaryKeyOptimizerUtil.java 0b59c9c

  src/org/apache/pig/data/SelfSpillBag.java d17f0a8 
  src/org/apache/pig/impl/plan/OperatorPlan.java 8b2e2e7 
  src/org/apache/pig/impl/util/UDFContext.java 09afc0a 
  src/org/apache/pig/tools/pigstats/PigStatsUtil.java 542cc2e 
  src/org/apache/pig/tools/pigstats/spark/SparkCounter.java PRE-CREATION 
  src/org/apache/pig/tools/pigstats/spark/SparkCounterGroup.java PRE-CREATION 
  src/org/apache/pig/tools/pigstats/spark/SparkCounters.java PRE-CREATION 
  src/org/apache/pig/tools/pigstats/spark/SparkJobStats.java PRE-CREATION 
  src/org/apache/pig/tools/pigstats/spark/SparkPigStats.java PRE-CREATION 
  src/org/apache/pig/tools/pigstats/spark/SparkPigStatusReporter.java PRE-CREATION 
  src/org/apache/pig/tools/pigstats/spark/SparkScriptState.java PRE-CREATION 
  src/org/apache/pig/tools/pigstats/spark/SparkStatsUtil.java PRE-CREATION 
  test/e2e/pig/build.xml f7c38ba 
  test/e2e/pig/conf/spark.conf PRE-CREATION 
  test/e2e/pig/drivers/TestDriverPig.pm bf9c302 
  test/e2e/pig/tests/streaming.conf 18f2fb2 
  test/excluded-tests-spark PRE-CREATION 
  test/org/apache/pig/newplan/logical/relational/TestLocationInPhysicalPlan.java 94b34b3 
  test/org/apache/pig/spark/TestIndexedKey.java PRE-CREATION 
  test/org/apache/pig/spark/TestSecondarySortSpark.java PRE-CREATION 
  test/org/apache/pig/test/MiniGenericCluster.java 9347269 
  test/org/apache/pig/test/TestAssert.java 6d4b5c6 
  test/org/apache/pig/test/TestBuiltin.java fbc3f1e 
  test/org/apache/pig/test/TestCase.java c9bb2fa 
  test/org/apache/pig/test/TestCollectedGroup.java a958d33 
  test/org/apache/pig/test/TestCombiner.java df44293 
  test/org/apache/pig/test/TestCubeOperator.java de96e6c 
  test/org/apache/pig/test/TestEvalPipeline.java 9efde13 
  test/org/apache/pig/test/TestEvalPipeline2.java c8f51d7 
  test/org/apache/pig/test/TestEvalPipelineLocal.java c12d595 
  test/org/apache/pig/test/TestFinish.java f18c103 
  test/org/apache/pig/test/TestForEachNestedPlanLocal.java b0aa3a8 
  test/org/apache/pig/test/TestGrunt.java 9eaf298 
  test/org/apache/pig/test/TestHBaseStorage.java 8d2ad85 
  test/org/apache/pig/test/TestLimitVariable.java 53b9dae 
  test/org/apache/pig/test/TestMapSideCogroup.java 2c78b4a 
  test/org/apache/pig/test/TestMergeJoin.java f1a9608 
  test/org/apache/pig/test/TestMergeJoinOuter.java 81aee55 
  test/org/apache/pig/test/TestMultiQuery.java c32eab7 
  test/org/apache/pig/test/TestMultiQueryLocal.java b9ac035 
  test/org/apache/pig/test/TestNativeMapReduce.java c4f6573 
  test/org/apache/pig/test/TestNullConstant.java 3ea4509 
  test/org/apache/pig/test/TestPigRunner.java fde8609 
  test/org/apache/pig/test/TestPigServerLocal.java fbabd03 
  test/org/apache/pig/test/TestProjectRange.java 2e3e7b8 
  test/org/apache/pig/test/TestPruneColumn.java 3936332 
  test/org/apache/pig/test/TestRank1.java 9e4ef62 
  test/org/apache/pig/test/TestRank2.java fc802a9 
  test/org/apache/pig/test/TestRank3.java 43af10d 
  test/org/apache/pig/test/TestSecondarySort.java 8991010 
  test/org/apache/pig/test/TestSkewedJoin.java dba2241 
  test/org/apache/pig/test/TestStoreBase.java eb3b253 
  test/org/apache/pig/test/Util.java 36d01e8 
  test/spark-tests PRE-CREATION 

Diff: https://reviews.apache.org/r/45667/diff/


Testing
-------

New UTs were added where required and ensure old UTs pass -> https://builds.apache.org/job/Pig-spark/


Thanks,

Pallavi Rao


Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message