crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject [43/43] git commit: CRUNCH-196: crunch -> crunch-core rename to fix build issues
Date Tue, 23 Apr 2013 20:41:45 GMT
CRUNCH-196: crunch -> crunch-core rename to fix build issues


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/890e0086
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/890e0086
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/890e0086

Branch: refs/heads/master
Commit: 890e0086a12df5006a23cfdd86f3703f929cb147
Parents: cbc7c2f
Author: Josh Wills <jwills@cloudera.com>
Authored: Tue Apr 23 13:37:16 2013 -0700
Committer: Josh Wills <jwills@cloudera.com>
Committed: Tue Apr 23 13:39:00 2013 -0700

----------------------------------------------------------------------
 crunch-contrib/pom.xml                             |    2 +-
 crunch-core/pom.xml                                |  182 +
 .../it/java/org/apache/crunch/CancelJobsIT.java    |   84 +
 .../src/it/java/org/apache/crunch/CleanTextIT.java |   82 +
 .../org/apache/crunch/CollectionPObjectIT.java     |   98 +
 .../it/java/org/apache/crunch/CollectionsIT.java   |  117 +
 .../org/apache/crunch/CollectionsLengthIT.java     |   70 +
 .../org/apache/crunch/DeepCopyCustomTuplesIT.java  |   79 +
 .../src/it/java/org/apache/crunch/EnumPairIT.java  |   59 +
 .../org/apache/crunch/FirstElementPObjectIT.java   |   61 +
 .../apache/crunch/IterableReuseProtectionIT.java   |   89 +
 .../it/java/org/apache/crunch/MRPipelineIT.java    |   78 +
 .../it/java/org/apache/crunch/MapPObjectIT.java    |  101 +
 .../src/it/java/org/apache/crunch/MapsIT.java      |  101 +
 .../it/java/org/apache/crunch/MaterializeIT.java   |  139 +
 .../java/org/apache/crunch/MaterializeToMapIT.java |   81 +
 .../java/org/apache/crunch/MultipleOutputIT.java   |  175 +
 .../org/apache/crunch/PCollectionGetSizeIT.java    |  151 +
 .../src/it/java/org/apache/crunch/PObjectsIT.java  |   99 +
 .../java/org/apache/crunch/PTableKeyValueIT.java   |  103 +
 .../src/it/java/org/apache/crunch/PageRankIT.java  |  168 +
 .../org/apache/crunch/StageResultsCountersIT.java  |  135 +
 .../it/java/org/apache/crunch/TermFrequencyIT.java |  135 +
 .../src/it/java/org/apache/crunch/TextPairIT.java  |   72 +
 .../src/it/java/org/apache/crunch/TfIdfIT.java     |  224 +
 .../org/apache/crunch/TupleNClassCastBugIT.java    |   95 +
 .../org/apache/crunch/UnionFromSameSourceIT.java   |  132 +
 .../src/it/java/org/apache/crunch/UnionIT.java     |  136 +
 .../it/java/org/apache/crunch/UnionResultsIT.java  |   80 +
 .../src/it/java/org/apache/crunch/WordCountIT.java |  171 +
 .../java/org/apache/crunch/fn/AggregatorsIT.java   |   83 +
 .../crunch/impl/mem/MemPipelineFileWritingIT.java  |   58 +
 .../crunch/impl/mr/collect/UnionCollectionIT.java  |  154 +
 .../apache/crunch/io/CompositePathIterableIT.java  |   84 +
 .../it/java/org/apache/crunch/io/NLineInputIT.java |   72 +
 .../java/org/apache/crunch/io/TextFileTableIT.java |   56 +
 .../crunch/io/avro/AvroFileSourceTargetIT.java     |  140 +
 .../org/apache/crunch/io/avro/AvroPipelineIT.java  |   95 +
 .../org/apache/crunch/io/avro/AvroReflectIT.java   |  109 +
 .../org/apache/crunch/io/avro/AvroWritableIT.java  |   89 +
 .../it/java/org/apache/crunch/lib/AggregateIT.java |  231 +
 .../java/org/apache/crunch/lib/AvroTypeSortIT.java |  145 +
 .../it/java/org/apache/crunch/lib/CogroupIT.java   |  112 +
 .../org/apache/crunch/lib/SecondarySortIT.java     |   65 +
 .../src/it/java/org/apache/crunch/lib/SetIT.java   |  114 +
 .../java/org/apache/crunch/lib/SortByValueIT.java  |   84 +
 .../src/it/java/org/apache/crunch/lib/SortIT.java  |  327 +
 .../apache/crunch/lib/SpecificAvroGroupByIT.java   |  119 +
 .../apache/crunch/lib/join/FullOuterJoinIT.java    |   51 +
 .../org/apache/crunch/lib/join/InnerJoinIT.java    |   51 +
 .../org/apache/crunch/lib/join/JoinTester.java     |  108 +
 .../apache/crunch/lib/join/LeftOuterJoinIT.java    |   51 +
 .../org/apache/crunch/lib/join/MapsideJoinIT.java  |  158 +
 .../crunch/lib/join/MultiAvroSchemaJoinIT.java     |  121 +
 .../apache/crunch/lib/join/RightOuterJoinIT.java   |   51 +
 .../org/apache/crunch/test/TemporaryPaths.java     |   40 +
 .../src/it/java/org/apache/crunch/test/Tests.java  |  124 +
 crunch-core/src/it/resources/customers.txt         |    4 +
 crunch-core/src/it/resources/docs.txt              |    6 +
 crunch-core/src/it/resources/letters.txt           |    2 +
 crunch-core/src/it/resources/log4j.properties      |   29 +
 crunch-core/src/it/resources/maugham.txt           |29112 +++++++++++++++
 crunch-core/src/it/resources/orders.txt            |    4 +
 .../org/apache/crunch/UnionITData/src1.txt         |    5 +
 .../org/apache/crunch/UnionITData/src2.txt         |    3 +
 .../apache/crunch/fn/AggregatorsITData/ints.txt    |    5 +
 .../org/apache/crunch/lib/CogroupITData/src1.txt   |    4 +
 .../org/apache/crunch/lib/CogroupITData/src2.txt   |    4 +
 .../src/it/resources/secondary_sort_input.txt      |    7 +
 crunch-core/src/it/resources/set1.txt              |    4 +
 crunch-core/src/it/resources/set2.txt              |    3 +
 crunch-core/src/it/resources/shakes.txt            | 3667 ++
 crunch-core/src/it/resources/sort_by_value.txt     |    5 +
 crunch-core/src/it/resources/urls.txt              |   11 +
 .../main/java/org/apache/crunch/Aggregator.java    |   86 +
 .../src/main/java/org/apache/crunch/CombineFn.java | 1211 +
 .../org/apache/crunch/CrunchRuntimeException.java  |   54 +
 .../src/main/java/org/apache/crunch/DoFn.java      |  162 +
 .../src/main/java/org/apache/crunch/Emitter.java   |   37 +
 .../src/main/java/org/apache/crunch/FilterFn.java  |  244 +
 .../java/org/apache/crunch/GroupingOptions.java    |  167 +
 .../src/main/java/org/apache/crunch/MapFn.java     |   41 +
 .../main/java/org/apache/crunch/PCollection.java   |  245 +
 .../main/java/org/apache/crunch/PGroupedTable.java |   53 +
 .../src/main/java/org/apache/crunch/PObject.java   |   36 +
 .../src/main/java/org/apache/crunch/PTable.java    |  181 +
 .../src/main/java/org/apache/crunch/Pair.java      |  105 +
 .../java/org/apache/crunch/ParallelDoOptions.java  |   62 +
 .../src/main/java/org/apache/crunch/Pipeline.java  |  138 +
 .../java/org/apache/crunch/PipelineExecution.java  |   54 +
 .../java/org/apache/crunch/PipelineResult.java     |   76 +
 .../src/main/java/org/apache/crunch/Source.java    |   52 +
 .../main/java/org/apache/crunch/SourceTarget.java  |   26 +
 .../main/java/org/apache/crunch/TableSource.java   |   28 +
 .../java/org/apache/crunch/TableSourceTarget.java  |   25 +
 .../src/main/java/org/apache/crunch/Target.java    |   83 +
 .../src/main/java/org/apache/crunch/Tuple.java     |   36 +
 .../src/main/java/org/apache/crunch/Tuple3.java    |   96 +
 .../src/main/java/org/apache/crunch/Tuple4.java    |  105 +
 .../src/main/java/org/apache/crunch/TupleN.java    |   73 +
 .../java/org/apache/crunch/fn/Aggregators.java     | 1111 +
 .../java/org/apache/crunch/fn/CompositeMapFn.java  |   71 +
 .../java/org/apache/crunch/fn/ExtractKeyFn.java    |   50 +
 .../main/java/org/apache/crunch/fn/FilterFns.java  |  112 +
 .../main/java/org/apache/crunch/fn/IdentityFn.java |   39 +
 .../main/java/org/apache/crunch/fn/MapKeysFn.java  |   32 +
 .../java/org/apache/crunch/fn/MapValuesFn.java     |   32 +
 .../main/java/org/apache/crunch/fn/PairMapFn.java  |   65 +
 .../java/org/apache/crunch/fn/package-info.java    |   22 +
 .../mapreduce/TaskAttemptContextFactory.java       |   70 +
 .../lib/jobcontrol/CrunchControlledJob.java        |  325 +
 .../mapreduce/lib/jobcontrol/CrunchJobControl.java |  211 +
 .../org/apache/crunch/impl/SingleUseIterable.java  |   49 +
 .../org/apache/crunch/impl/mem/MemPipeline.java    |  275 +
 .../crunch/impl/mem/collect/MemCollection.java     |  295 +
 .../crunch/impl/mem/collect/MemGroupedTable.java   |  113 +
 .../apache/crunch/impl/mem/collect/MemTable.java   |  177 +
 .../apache/crunch/impl/mem/collect/Shuffler.java   |  149 +
 .../crunch/impl/mem/emit/InMemoryEmitter.java      |   57 +
 .../org/apache/crunch/impl/mem/package-info.java   |   22 +
 .../java/org/apache/crunch/impl/mr/MRPipeline.java |  396 +
 .../crunch/impl/mr/collect/DoCollectionImpl.java   |   74 +
 .../apache/crunch/impl/mr/collect/DoTableImpl.java |   84 +
 .../crunch/impl/mr/collect/InputCollection.java    |   85 +
 .../apache/crunch/impl/mr/collect/InputTable.java  |   86 +
 .../crunch/impl/mr/collect/PCollectionImpl.java    |  295 +
 .../crunch/impl/mr/collect/PGroupedTableImpl.java  |  144 +
 .../apache/crunch/impl/mr/collect/PTableBase.java  |  169 +
 .../crunch/impl/mr/collect/UnionCollection.java    |   80 +
 .../apache/crunch/impl/mr/collect/UnionTable.java  |   92 +
 .../crunch/impl/mr/emit/IntermediateEmitter.java   |   64 +
 .../crunch/impl/mr/emit/MultipleOutputEmitter.java |   56 +
 .../apache/crunch/impl/mr/emit/OutputEmitter.java  |   52 +
 .../impl/mr/exec/CappedExponentialCounter.java     |   40 +
 .../apache/crunch/impl/mr/exec/CrunchJobHooks.java |  153 +
 .../org/apache/crunch/impl/mr/exec/MRExecutor.java |  198 +
 .../org/apache/crunch/impl/mr/package-info.java    |   22 +
 .../org/apache/crunch/impl/mr/plan/DoNode.java     |  163 +
 .../apache/crunch/impl/mr/plan/DotfileWriter.java  |  238 +
 .../java/org/apache/crunch/impl/mr/plan/Edge.java  |  125 +
 .../java/org/apache/crunch/impl/mr/plan/Graph.java |  133 +
 .../apache/crunch/impl/mr/plan/GraphBuilder.java   |   92 +
 .../apache/crunch/impl/mr/plan/JobNameBuilder.java |   79 +
 .../apache/crunch/impl/mr/plan/JobPrototype.java   |  245 +
 .../crunch/impl/mr/plan/MSCROutputHandler.java     |   77 +
 .../apache/crunch/impl/mr/plan/MSCRPlanner.java    |  378 +
 .../org/apache/crunch/impl/mr/plan/NodePath.java   |  124 +
 .../crunch/impl/mr/plan/PlanningParameters.java    |   38 +
 .../org/apache/crunch/impl/mr/plan/Vertex.java     |  126 +
 .../apache/crunch/impl/mr/run/CrunchCombiner.java  |   27 +
 .../crunch/impl/mr/run/CrunchInputFormat.java      |   78 +
 .../crunch/impl/mr/run/CrunchInputSplit.java       |  116 +
 .../apache/crunch/impl/mr/run/CrunchMapper.java    |   73 +
 .../crunch/impl/mr/run/CrunchRecordReader.java     |   75 +
 .../apache/crunch/impl/mr/run/CrunchReducer.java   |   73 +
 .../crunch/impl/mr/run/CrunchTaskContext.java      |   86 +
 .../org/apache/crunch/impl/mr/run/NodeContext.java |   35 +
 .../java/org/apache/crunch/impl/mr/run/RTNode.java |  124 +
 .../crunch/impl/mr/run/RuntimeParameters.java      |   38 +
 .../src/main/java/org/apache/crunch/io/At.java     |  257 +
 .../apache/crunch/io/CompositePathIterable.java    |  102 +
 .../java/org/apache/crunch/io/CrunchInputs.java    |   71 +
 .../java/org/apache/crunch/io/CrunchOutputs.java   |  184 +
 .../org/apache/crunch/io/FileNamingScheme.java     |   58 +
 .../org/apache/crunch/io/FileReaderFactory.java    |   27 +
 .../java/org/apache/crunch/io/FormatBundle.java    |  121 +
 .../src/main/java/org/apache/crunch/io/From.java   |  324 +
 .../java/org/apache/crunch/io/MapReduceTarget.java |   27 +
 .../java/org/apache/crunch/io/OutputHandler.java   |   25 +
 .../main/java/org/apache/crunch/io/PathTarget.java |   36 +
 .../java/org/apache/crunch/io/PathTargetImpl.java  |   64 +
 .../java/org/apache/crunch/io/ReadableSource.java  |   41 +
 .../org/apache/crunch/io/ReadableSourceTarget.java |   30 +
 .../crunch/io/SequentialFileNamingScheme.java      |   51 +
 .../org/apache/crunch/io/SourceTargetHelper.java   |   48 +
 .../src/main/java/org/apache/crunch/io/To.java     |  153 +
 .../crunch/io/avro/AvroFileReaderFactory.java      |   96 +
 .../org/apache/crunch/io/avro/AvroFileSource.java  |   58 +
 .../crunch/io/avro/AvroFileSourceTarget.java       |   39 +
 .../org/apache/crunch/io/avro/AvroFileTarget.java  |   91 +
 .../apache/crunch/io/impl/AutoClosingIterator.java |   62 +
 .../org/apache/crunch/io/impl/FileSourceImpl.java  |  104 +
 .../apache/crunch/io/impl/FileTableSourceImpl.java |   41 +
 .../org/apache/crunch/io/impl/FileTargetImpl.java  |  162 +
 .../io/impl/ReadableSourcePathTargetImpl.java      |   39 +
 .../crunch/io/impl/ReadableSourceTargetImpl.java   |   37 +
 .../crunch/io/impl/SourcePathTargetImpl.java       |   50 +
 .../apache/crunch/io/impl/SourceTargetImpl.java    |   89 +
 .../crunch/io/impl/TableSourcePathTargetImpl.java  |   41 +
 .../crunch/io/impl/TableSourceTargetImpl.java      |   35 +
 .../java/org/apache/crunch/io/package-info.java    |   22 +
 .../org/apache/crunch/io/seq/SeqFileHelper.java    |   35 +
 .../apache/crunch/io/seq/SeqFileReaderFactory.java |  112 +
 .../org/apache/crunch/io/seq/SeqFileSource.java    |   47 +
 .../apache/crunch/io/seq/SeqFileSourceTarget.java  |   44 +
 .../apache/crunch/io/seq/SeqFileTableSource.java   |   57 +
 .../crunch/io/seq/SeqFileTableSourceTarget.java    |   54 +
 .../org/apache/crunch/io/seq/SeqFileTarget.java    |   55 +
 .../crunch/io/text/BZip2TextInputFormat.java       |  235 +
 .../apache/crunch/io/text/CBZip2InputStream.java   |  980 +
 .../java/org/apache/crunch/io/text/LineParser.java |  125 +
 .../org/apache/crunch/io/text/NLineFileSource.java |   77 +
 .../crunch/io/text/TextFileReaderFactory.java      |   83 +
 .../org/apache/crunch/io/text/TextFileSource.java  |   73 +
 .../crunch/io/text/TextFileSourceTarget.java       |   44 +
 .../apache/crunch/io/text/TextFileTableSource.java |   81 +
 .../crunch/io/text/TextFileTableSourceTarget.java  |   63 +
 .../org/apache/crunch/io/text/TextFileTarget.java  |  109 +
 .../main/java/org/apache/crunch/lib/Aggregate.java |  272 +
 .../main/java/org/apache/crunch/lib/Cartesian.java |  216 +
 .../main/java/org/apache/crunch/lib/Cogroup.java   |  106 +
 .../main/java/org/apache/crunch/lib/Distinct.java  |  126 +
 .../src/main/java/org/apache/crunch/lib/Join.java  |  181 +
 .../main/java/org/apache/crunch/lib/PTables.java   |  117 +
 .../main/java/org/apache/crunch/lib/Sample.java    |  217 +
 .../java/org/apache/crunch/lib/SampleUtils.java    |  168 +
 .../java/org/apache/crunch/lib/SecondarySort.java  |  118 +
 .../src/main/java/org/apache/crunch/lib/Set.java   |  118 +
 .../src/main/java/org/apache/crunch/lib/Sort.java  |  294 +
 .../apache/crunch/lib/join/FullOuterJoinFn.java    |  102 +
 .../org/apache/crunch/lib/join/InnerJoinFn.java    |   78 +
 .../java/org/apache/crunch/lib/join/JoinFn.java    |   81 +
 .../java/org/apache/crunch/lib/join/JoinUtils.java |  126 +
 .../apache/crunch/lib/join/LeftOuterJoinFn.java    |   98 +
 .../org/apache/crunch/lib/join/MapsideJoin.java    |  164 +
 .../apache/crunch/lib/join/RightOuterJoinFn.java   |   83 +
 .../org/apache/crunch/lib/join/package-info.java   |   22 +
 .../java/org/apache/crunch/lib/package-info.java   |   22 +
 .../org/apache/crunch/lib/sort/Comparators.java    |  187 +
 .../java/org/apache/crunch/lib/sort/SortFns.java   |  210 +
 .../crunch/lib/sort/TotalOrderPartitioner.java     |  145 +
 .../crunch/materialize/MaterializableIterable.java |   81 +
 .../crunch/materialize/MaterializableMap.java      |   50 +
 .../materialize/pobject/CollectionPObject.java     |   55 +
 .../materialize/pobject/FirstElementPObject.java   |   50 +
 .../crunch/materialize/pobject/MapPObject.java     |   62 +
 .../crunch/materialize/pobject/PObjectImpl.java    |   85 +
 .../main/java/org/apache/crunch/package-info.java  |   25 +
 .../apache/crunch/types/CollectionDeepCopier.java  |   57 +
 .../java/org/apache/crunch/types/Converter.java    |   41 +
 .../java/org/apache/crunch/types/DeepCopier.java   |   60 +
 .../org/apache/crunch/types/MapDeepCopier.java     |   54 +
 .../org/apache/crunch/types/PGroupedTableType.java |  141 +
 .../java/org/apache/crunch/types/PTableType.java   |   44 +
 .../main/java/org/apache/crunch/types/PType.java   |   86 +
 .../java/org/apache/crunch/types/PTypeFamily.java  |   77 +
 .../java/org/apache/crunch/types/PTypeUtils.java   |   66 +
 .../main/java/org/apache/crunch/types/PTypes.java  |  252 +
 .../main/java/org/apache/crunch/types/Protos.java  |  173 +
 .../org/apache/crunch/types/TupleDeepCopier.java   |   65 +
 .../java/org/apache/crunch/types/TupleFactory.java |  134 +
 .../apache/crunch/types/avro/AvroCapabilities.java |  106 +
 .../apache/crunch/types/avro/AvroDeepCopier.java   |  209 +
 .../crunch/types/avro/AvroGroupedTableType.java    |  114 +
 .../apache/crunch/types/avro/AvroInputFormat.java  |   41 +
 .../apache/crunch/types/avro/AvroKeyConverter.java |   65 +
 .../apache/crunch/types/avro/AvroOutputFormat.java |   87 +
 .../crunch/types/avro/AvroPairConverter.java       |  108 +
 .../apache/crunch/types/avro/AvroRecordReader.java |  114 +
 .../apache/crunch/types/avro/AvroTableType.java    |  151 +
 .../crunch/types/avro/AvroTextOutputFormat.java    |   60 +
 .../org/apache/crunch/types/avro/AvroType.java     |  199 +
 .../apache/crunch/types/avro/AvroTypeFamily.java   |  164 +
 .../crunch/types/avro/AvroUtf8InputFormat.java     |   98 +
 .../java/org/apache/crunch/types/avro/Avros.java   |  709 +
 .../crunch/types/avro/ReflectDataFactory.java      |   41 +
 .../crunch/types/avro/SafeAvroSerialization.java   |  145 +
 .../org/apache/crunch/types/avro/package-info.java |   22 +
 .../java/org/apache/crunch/types/package-info.java |   22 +
 .../types/writable/GenericArrayWritable.java       |  135 +
 .../crunch/types/writable/TextMapWritable.java     |   88 +
 .../crunch/types/writable/TupleWritable.java       |  224 +
 .../crunch/types/writable/WritableDeepCopier.java  |   70 +
 .../types/writable/WritableGroupedTableType.java   |   85 +
 .../types/writable/WritablePairConverter.java      |   62 +
 .../crunch/types/writable/WritableTableType.java   |  130 +
 .../apache/crunch/types/writable/WritableType.java |  133 +
 .../crunch/types/writable/WritableTypeFamily.java  |  147 +
 .../types/writable/WritableValueConverter.java     |   60 +
 .../apache/crunch/types/writable/Writables.java    |  588 +
 .../apache/crunch/types/writable/package-info.java |   22 +
 .../java/org/apache/crunch/util/CrunchTool.java    |  118 +
 .../java/org/apache/crunch/util/DistCache.java     |  231 +
 .../org/apache/crunch/util/PartitionUtils.java     |   34 +
 .../main/java/org/apache/crunch/util/Tuples.java   |  150 +
 .../java/org/apache/crunch/util/package-info.java  |   22 +
 crunch-core/src/main/resources/log4j.properties    |   24 +
 crunch-core/src/site/site.xml                      |   34 +
 crunch-core/src/test/avro/employee.avsc            |   26 +
 crunch-core/src/test/avro/person.avsc              |   26 +
 .../src/test/java/org/apache/crunch/AndFnTest.java |   77 +
 .../test/java/org/apache/crunch/CombineFnTest.java |  222 +
 .../src/test/java/org/apache/crunch/NotFnTest.java |   72 +
 .../src/test/java/org/apache/crunch/OrFnTest.java  |   78 +
 .../src/test/java/org/apache/crunch/PairTest.java  |   66 +
 .../src/test/java/org/apache/crunch/TupleTest.java |  139 +
 .../test/java/org/apache/crunch/WriteModeTest.java |  103 +
 .../java/org/apache/crunch/fn/AggregatorsTest.java |  239 +
 .../org/apache/crunch/fn/ExtractKeyFnTest.java     |   44 +
 .../java/org/apache/crunch/fn/FilterFnTest.java    |   85 +
 .../java/org/apache/crunch/fn/MapKeysTest.java     |   51 +
 .../java/org/apache/crunch/fn/MapValuesTest.java   |   50 +
 .../java/org/apache/crunch/fn/PairMapTest.java     |   52 +
 .../org/apache/crunch/fn/StoreLastEmitter.java     |   41 +
 .../apache/crunch/impl/SingleUseIterableTest.java  |   54 +
 .../org/apache/crunch/impl/mr/MRPipelineTest.java  |   86 +
 .../impl/mr/collect/DoCollectionImplTest.java      |  112 +
 .../crunch/impl/mr/collect/DoTableImplTest.java    |   86 +
 .../impl/mr/emit/IntermediateEmitterTest.java      |   83 +
 .../impl/mr/exec/CappedExponentialCounterTest.java |   42 +
 .../crunch/impl/mr/exec/CrunchJobHooksTest.java    |   42 +
 .../crunch/impl/mr/plan/DotfileWriterTest.java     |  132 +
 .../crunch/impl/mr/plan/JobNameBuilderTest.java    |   41 +
 .../crunch/io/SequentialFileNamingSchemeTest.java  |   84 +
 .../apache/crunch/io/SourceTargetHelperTest.java   |   59 +
 .../crunch/io/avro/AvroFileReaderFactoryTest.java  |  184 +
 .../apache/crunch/io/avro/AvroFileSourceTest.java  |   91 +
 .../lib/AvroIndexedRecordPartitionerTest.java      |   98 +
 .../java/org/apache/crunch/lib/CartesianTest.java  |   77 +
 .../java/org/apache/crunch/lib/DistinctTest.java   |   52 +
 .../java/org/apache/crunch/lib/SampleTest.java     |   71 +
 .../org/apache/crunch/lib/SecondarySortTest.java   |   53 +
 .../crunch/lib/TupleWritablePartitionerTest.java   |   68 +
 .../lib/join/BrokenLeftAndOuterJoinTest.java       |   90 +
 .../crunch/lib/join/FullOuterJoinFnTest.java       |   48 +
 .../apache/crunch/lib/join/InnerJoinFnTest.java    |   42 +
 .../org/apache/crunch/lib/join/JoinFnTestBase.java |   82 +
 .../apache/crunch/lib/join/LeftOuterJoinTest.java  |   46 +
 .../crunch/lib/join/RightOuterJoinFnTest.java      |   46 +
 .../java/org/apache/crunch/test/CountersTest.java  |   70 +
 .../java/org/apache/crunch/test/StringWrapper.java |  102 +
 .../crunch/types/CollectionDeepCopierTest.java     |   61 +
 .../org/apache/crunch/types/MapDeepCopierTest.java |   63 +
 .../org/apache/crunch/types/PTypeUtilsTest.java    |   89 +
 .../java/org/apache/crunch/types/PTypesTest.java   |   34 +
 .../apache/crunch/types/TupleDeepCopierTest.java   |   77 +
 .../org/apache/crunch/types/TupleFactoryTest.java  |   69 +
 .../crunch/types/avro/AvroDeepCopierTest.java      |  107 +
 .../types/avro/AvroGroupedTableTypeTest.java       |   60 +
 .../crunch/types/avro/AvroTableTypeTest.java       |   72 +
 .../org/apache/crunch/types/avro/AvroTypeTest.java |  279 +
 .../org/apache/crunch/types/avro/AvrosTest.java    |  325 +
 .../types/writable/GenericArrayWritableTest.java   |   70 +
 .../types/writable/WritableDeepCopierTest.java     |   54 +
 .../writable/WritableGroupedTableTypeTest.java     |   56 +
 .../types/writable/WritableTableTypeTest.java      |   47 +
 .../crunch/types/writable/WritableTypeTest.java    |   97 +
 .../crunch/types/writable/WritablesTest.java       |  256 +
 .../java/org/apache/crunch/util/DistCacheTest.java |  156 +
 crunch-dist/pom.xml                                |    2 +-
 crunch-examples/pom.xml                            |    2 +-
 crunch-hbase/pom.xml                               |    2 +-
 crunch-scrunch/pom.xml                             |    2 +-
 crunch/pom.xml                                     |  182 -
 .../it/java/org/apache/crunch/CancelJobsIT.java    |   84 -
 .../src/it/java/org/apache/crunch/CleanTextIT.java |   82 -
 .../org/apache/crunch/CollectionPObjectIT.java     |   98 -
 .../it/java/org/apache/crunch/CollectionsIT.java   |  117 -
 .../org/apache/crunch/CollectionsLengthIT.java     |   70 -
 .../org/apache/crunch/DeepCopyCustomTuplesIT.java  |   79 -
 .../src/it/java/org/apache/crunch/EnumPairIT.java  |   59 -
 .../org/apache/crunch/FirstElementPObjectIT.java   |   61 -
 .../apache/crunch/IterableReuseProtectionIT.java   |   89 -
 .../it/java/org/apache/crunch/MRPipelineIT.java    |   78 -
 .../it/java/org/apache/crunch/MapPObjectIT.java    |  101 -
 crunch/src/it/java/org/apache/crunch/MapsIT.java   |  101 -
 .../it/java/org/apache/crunch/MaterializeIT.java   |  139 -
 .../java/org/apache/crunch/MaterializeToMapIT.java |   81 -
 .../java/org/apache/crunch/MultipleOutputIT.java   |  175 -
 .../org/apache/crunch/PCollectionGetSizeIT.java    |  151 -
 .../src/it/java/org/apache/crunch/PObjectsIT.java  |   99 -
 .../java/org/apache/crunch/PTableKeyValueIT.java   |  103 -
 .../src/it/java/org/apache/crunch/PageRankIT.java  |  168 -
 .../org/apache/crunch/StageResultsCountersIT.java  |  135 -
 .../it/java/org/apache/crunch/TermFrequencyIT.java |  135 -
 .../src/it/java/org/apache/crunch/TextPairIT.java  |   72 -
 crunch/src/it/java/org/apache/crunch/TfIdfIT.java  |  224 -
 .../org/apache/crunch/TupleNClassCastBugIT.java    |   95 -
 .../org/apache/crunch/UnionFromSameSourceIT.java   |  132 -
 crunch/src/it/java/org/apache/crunch/UnionIT.java  |  136 -
 .../it/java/org/apache/crunch/UnionResultsIT.java  |   80 -
 .../src/it/java/org/apache/crunch/WordCountIT.java |  171 -
 .../java/org/apache/crunch/fn/AggregatorsIT.java   |   83 -
 .../crunch/impl/mem/MemPipelineFileWritingIT.java  |   58 -
 .../crunch/impl/mr/collect/UnionCollectionIT.java  |  154 -
 .../apache/crunch/io/CompositePathIterableIT.java  |   84 -
 .../it/java/org/apache/crunch/io/NLineInputIT.java |   72 -
 .../java/org/apache/crunch/io/TextFileTableIT.java |   56 -
 .../crunch/io/avro/AvroFileSourceTargetIT.java     |  140 -
 .../org/apache/crunch/io/avro/AvroPipelineIT.java  |   95 -
 .../org/apache/crunch/io/avro/AvroReflectIT.java   |  109 -
 .../org/apache/crunch/io/avro/AvroWritableIT.java  |   89 -
 .../it/java/org/apache/crunch/lib/AggregateIT.java |  231 -
 .../java/org/apache/crunch/lib/AvroTypeSortIT.java |  145 -
 .../it/java/org/apache/crunch/lib/CogroupIT.java   |  112 -
 .../org/apache/crunch/lib/SecondarySortIT.java     |   65 -
 .../src/it/java/org/apache/crunch/lib/SetIT.java   |  114 -
 .../java/org/apache/crunch/lib/SortByValueIT.java  |   84 -
 .../src/it/java/org/apache/crunch/lib/SortIT.java  |  327 -
 .../apache/crunch/lib/SpecificAvroGroupByIT.java   |  119 -
 .../apache/crunch/lib/join/FullOuterJoinIT.java    |   51 -
 .../org/apache/crunch/lib/join/InnerJoinIT.java    |   51 -
 .../org/apache/crunch/lib/join/JoinTester.java     |  108 -
 .../apache/crunch/lib/join/LeftOuterJoinIT.java    |   51 -
 .../org/apache/crunch/lib/join/MapsideJoinIT.java  |  158 -
 .../crunch/lib/join/MultiAvroSchemaJoinIT.java     |  121 -
 .../apache/crunch/lib/join/RightOuterJoinIT.java   |   51 -
 .../org/apache/crunch/test/TemporaryPaths.java     |   40 -
 .../src/it/java/org/apache/crunch/test/Tests.java  |  124 -
 crunch/src/it/resources/customers.txt              |    4 -
 crunch/src/it/resources/docs.txt                   |    6 -
 crunch/src/it/resources/letters.txt                |    2 -
 crunch/src/it/resources/log4j.properties           |   29 -
 crunch/src/it/resources/maugham.txt                |29112 ---------------
 crunch/src/it/resources/orders.txt                 |    4 -
 .../org/apache/crunch/UnionITData/src1.txt         |    5 -
 .../org/apache/crunch/UnionITData/src2.txt         |    3 -
 .../apache/crunch/fn/AggregatorsITData/ints.txt    |    5 -
 .../org/apache/crunch/lib/CogroupITData/src1.txt   |    4 -
 .../org/apache/crunch/lib/CogroupITData/src2.txt   |    4 -
 crunch/src/it/resources/secondary_sort_input.txt   |    7 -
 crunch/src/it/resources/set1.txt                   |    4 -
 crunch/src/it/resources/set2.txt                   |    3 -
 crunch/src/it/resources/shakes.txt                 | 3667 --
 crunch/src/it/resources/sort_by_value.txt          |    5 -
 crunch/src/it/resources/urls.txt                   |   11 -
 .../main/java/org/apache/crunch/Aggregator.java    |   86 -
 .../src/main/java/org/apache/crunch/CombineFn.java | 1211 -
 .../org/apache/crunch/CrunchRuntimeException.java  |   54 -
 crunch/src/main/java/org/apache/crunch/DoFn.java   |  162 -
 .../src/main/java/org/apache/crunch/Emitter.java   |   37 -
 .../src/main/java/org/apache/crunch/FilterFn.java  |  244 -
 .../java/org/apache/crunch/GroupingOptions.java    |  167 -
 crunch/src/main/java/org/apache/crunch/MapFn.java  |   41 -
 .../main/java/org/apache/crunch/PCollection.java   |  245 -
 .../main/java/org/apache/crunch/PGroupedTable.java |   53 -
 .../src/main/java/org/apache/crunch/PObject.java   |   36 -
 crunch/src/main/java/org/apache/crunch/PTable.java |  181 -
 crunch/src/main/java/org/apache/crunch/Pair.java   |  105 -
 .../java/org/apache/crunch/ParallelDoOptions.java  |   62 -
 .../src/main/java/org/apache/crunch/Pipeline.java  |  138 -
 .../java/org/apache/crunch/PipelineExecution.java  |   54 -
 .../java/org/apache/crunch/PipelineResult.java     |   76 -
 crunch/src/main/java/org/apache/crunch/Source.java |   52 -
 .../main/java/org/apache/crunch/SourceTarget.java  |   26 -
 .../main/java/org/apache/crunch/TableSource.java   |   28 -
 .../java/org/apache/crunch/TableSourceTarget.java  |   25 -
 crunch/src/main/java/org/apache/crunch/Target.java |   83 -
 crunch/src/main/java/org/apache/crunch/Tuple.java  |   36 -
 crunch/src/main/java/org/apache/crunch/Tuple3.java |   96 -
 crunch/src/main/java/org/apache/crunch/Tuple4.java |  105 -
 crunch/src/main/java/org/apache/crunch/TupleN.java |   73 -
 .../java/org/apache/crunch/fn/Aggregators.java     | 1111 -
 .../java/org/apache/crunch/fn/CompositeMapFn.java  |   71 -
 .../java/org/apache/crunch/fn/ExtractKeyFn.java    |   50 -
 .../main/java/org/apache/crunch/fn/FilterFns.java  |  112 -
 .../main/java/org/apache/crunch/fn/IdentityFn.java |   39 -
 .../main/java/org/apache/crunch/fn/MapKeysFn.java  |   32 -
 .../java/org/apache/crunch/fn/MapValuesFn.java     |   32 -
 .../main/java/org/apache/crunch/fn/PairMapFn.java  |   65 -
 .../java/org/apache/crunch/fn/package-info.java    |   22 -
 .../mapreduce/TaskAttemptContextFactory.java       |   70 -
 .../lib/jobcontrol/CrunchControlledJob.java        |  325 -
 .../mapreduce/lib/jobcontrol/CrunchJobControl.java |  211 -
 .../org/apache/crunch/impl/SingleUseIterable.java  |   49 -
 .../org/apache/crunch/impl/mem/MemPipeline.java    |  275 -
 .../crunch/impl/mem/collect/MemCollection.java     |  295 -
 .../crunch/impl/mem/collect/MemGroupedTable.java   |  113 -
 .../apache/crunch/impl/mem/collect/MemTable.java   |  177 -
 .../apache/crunch/impl/mem/collect/Shuffler.java   |  149 -
 .../crunch/impl/mem/emit/InMemoryEmitter.java      |   57 -
 .../org/apache/crunch/impl/mem/package-info.java   |   22 -
 .../java/org/apache/crunch/impl/mr/MRPipeline.java |  396 -
 .../crunch/impl/mr/collect/DoCollectionImpl.java   |   74 -
 .../apache/crunch/impl/mr/collect/DoTableImpl.java |   84 -
 .../crunch/impl/mr/collect/InputCollection.java    |   85 -
 .../apache/crunch/impl/mr/collect/InputTable.java  |   86 -
 .../crunch/impl/mr/collect/PCollectionImpl.java    |  295 -
 .../crunch/impl/mr/collect/PGroupedTableImpl.java  |  144 -
 .../apache/crunch/impl/mr/collect/PTableBase.java  |  169 -
 .../crunch/impl/mr/collect/UnionCollection.java    |   80 -
 .../apache/crunch/impl/mr/collect/UnionTable.java  |   92 -
 .../crunch/impl/mr/emit/IntermediateEmitter.java   |   64 -
 .../crunch/impl/mr/emit/MultipleOutputEmitter.java |   56 -
 .../apache/crunch/impl/mr/emit/OutputEmitter.java  |   52 -
 .../impl/mr/exec/CappedExponentialCounter.java     |   40 -
 .../apache/crunch/impl/mr/exec/CrunchJobHooks.java |  153 -
 .../org/apache/crunch/impl/mr/exec/MRExecutor.java |  198 -
 .../org/apache/crunch/impl/mr/package-info.java    |   22 -
 .../org/apache/crunch/impl/mr/plan/DoNode.java     |  163 -
 .../apache/crunch/impl/mr/plan/DotfileWriter.java  |  238 -
 .../java/org/apache/crunch/impl/mr/plan/Edge.java  |  125 -
 .../java/org/apache/crunch/impl/mr/plan/Graph.java |  133 -
 .../apache/crunch/impl/mr/plan/GraphBuilder.java   |   92 -
 .../apache/crunch/impl/mr/plan/JobNameBuilder.java |   79 -
 .../apache/crunch/impl/mr/plan/JobPrototype.java   |  245 -
 .../crunch/impl/mr/plan/MSCROutputHandler.java     |   77 -
 .../apache/crunch/impl/mr/plan/MSCRPlanner.java    |  378 -
 .../org/apache/crunch/impl/mr/plan/NodePath.java   |  124 -
 .../crunch/impl/mr/plan/PlanningParameters.java    |   38 -
 .../org/apache/crunch/impl/mr/plan/Vertex.java     |  126 -
 .../apache/crunch/impl/mr/run/CrunchCombiner.java  |   27 -
 .../crunch/impl/mr/run/CrunchInputFormat.java      |   78 -
 .../crunch/impl/mr/run/CrunchInputSplit.java       |  116 -
 .../apache/crunch/impl/mr/run/CrunchMapper.java    |   73 -
 .../crunch/impl/mr/run/CrunchRecordReader.java     |   75 -
 .../apache/crunch/impl/mr/run/CrunchReducer.java   |   73 -
 .../crunch/impl/mr/run/CrunchTaskContext.java      |   86 -
 .../org/apache/crunch/impl/mr/run/NodeContext.java |   35 -
 .../java/org/apache/crunch/impl/mr/run/RTNode.java |  124 -
 .../crunch/impl/mr/run/RuntimeParameters.java      |   38 -
 crunch/src/main/java/org/apache/crunch/io/At.java  |  257 -
 .../apache/crunch/io/CompositePathIterable.java    |  102 -
 .../java/org/apache/crunch/io/CrunchInputs.java    |   71 -
 .../java/org/apache/crunch/io/CrunchOutputs.java   |  184 -
 .../org/apache/crunch/io/FileNamingScheme.java     |   58 -
 .../org/apache/crunch/io/FileReaderFactory.java    |   27 -
 .../java/org/apache/crunch/io/FormatBundle.java    |  121 -
 .../src/main/java/org/apache/crunch/io/From.java   |  324 -
 .../java/org/apache/crunch/io/MapReduceTarget.java |   27 -
 .../java/org/apache/crunch/io/OutputHandler.java   |   25 -
 .../main/java/org/apache/crunch/io/PathTarget.java |   36 -
 .../java/org/apache/crunch/io/PathTargetImpl.java  |   64 -
 .../java/org/apache/crunch/io/ReadableSource.java  |   41 -
 .../org/apache/crunch/io/ReadableSourceTarget.java |   30 -
 .../crunch/io/SequentialFileNamingScheme.java      |   51 -
 .../org/apache/crunch/io/SourceTargetHelper.java   |   48 -
 crunch/src/main/java/org/apache/crunch/io/To.java  |  153 -
 .../crunch/io/avro/AvroFileReaderFactory.java      |   96 -
 .../org/apache/crunch/io/avro/AvroFileSource.java  |   58 -
 .../crunch/io/avro/AvroFileSourceTarget.java       |   39 -
 .../org/apache/crunch/io/avro/AvroFileTarget.java  |   91 -
 .../apache/crunch/io/impl/AutoClosingIterator.java |   62 -
 .../org/apache/crunch/io/impl/FileSourceImpl.java  |  104 -
 .../apache/crunch/io/impl/FileTableSourceImpl.java |   41 -
 .../org/apache/crunch/io/impl/FileTargetImpl.java  |  162 -
 .../io/impl/ReadableSourcePathTargetImpl.java      |   39 -
 .../crunch/io/impl/ReadableSourceTargetImpl.java   |   37 -
 .../crunch/io/impl/SourcePathTargetImpl.java       |   50 -
 .../apache/crunch/io/impl/SourceTargetImpl.java    |   89 -
 .../crunch/io/impl/TableSourcePathTargetImpl.java  |   41 -
 .../crunch/io/impl/TableSourceTargetImpl.java      |   35 -
 .../java/org/apache/crunch/io/package-info.java    |   22 -
 .../org/apache/crunch/io/seq/SeqFileHelper.java    |   35 -
 .../apache/crunch/io/seq/SeqFileReaderFactory.java |  112 -
 .../org/apache/crunch/io/seq/SeqFileSource.java    |   47 -
 .../apache/crunch/io/seq/SeqFileSourceTarget.java  |   44 -
 .../apache/crunch/io/seq/SeqFileTableSource.java   |   57 -
 .../crunch/io/seq/SeqFileTableSourceTarget.java    |   54 -
 .../org/apache/crunch/io/seq/SeqFileTarget.java    |   55 -
 .../crunch/io/text/BZip2TextInputFormat.java       |  235 -
 .../apache/crunch/io/text/CBZip2InputStream.java   |  980 -
 .../java/org/apache/crunch/io/text/LineParser.java |  125 -
 .../org/apache/crunch/io/text/NLineFileSource.java |   77 -
 .../crunch/io/text/TextFileReaderFactory.java      |   83 -
 .../org/apache/crunch/io/text/TextFileSource.java  |   73 -
 .../crunch/io/text/TextFileSourceTarget.java       |   44 -
 .../apache/crunch/io/text/TextFileTableSource.java |   81 -
 .../crunch/io/text/TextFileTableSourceTarget.java  |   63 -
 .../org/apache/crunch/io/text/TextFileTarget.java  |  109 -
 .../main/java/org/apache/crunch/lib/Aggregate.java |  272 -
 .../main/java/org/apache/crunch/lib/Cartesian.java |  216 -
 .../main/java/org/apache/crunch/lib/Cogroup.java   |  106 -
 .../main/java/org/apache/crunch/lib/Distinct.java  |  126 -
 .../src/main/java/org/apache/crunch/lib/Join.java  |  181 -
 .../main/java/org/apache/crunch/lib/PTables.java   |  117 -
 .../main/java/org/apache/crunch/lib/Sample.java    |  217 -
 .../java/org/apache/crunch/lib/SampleUtils.java    |  168 -
 .../java/org/apache/crunch/lib/SecondarySort.java  |  118 -
 .../src/main/java/org/apache/crunch/lib/Set.java   |  118 -
 .../src/main/java/org/apache/crunch/lib/Sort.java  |  294 -
 .../apache/crunch/lib/join/FullOuterJoinFn.java    |  102 -
 .../org/apache/crunch/lib/join/InnerJoinFn.java    |   78 -
 .../java/org/apache/crunch/lib/join/JoinFn.java    |   81 -
 .../java/org/apache/crunch/lib/join/JoinUtils.java |  126 -
 .../apache/crunch/lib/join/LeftOuterJoinFn.java    |   98 -
 .../org/apache/crunch/lib/join/MapsideJoin.java    |  164 -
 .../apache/crunch/lib/join/RightOuterJoinFn.java   |   83 -
 .../org/apache/crunch/lib/join/package-info.java   |   22 -
 .../java/org/apache/crunch/lib/package-info.java   |   22 -
 .../org/apache/crunch/lib/sort/Comparators.java    |  187 -
 .../java/org/apache/crunch/lib/sort/SortFns.java   |  210 -
 .../crunch/lib/sort/TotalOrderPartitioner.java     |  145 -
 .../crunch/materialize/MaterializableIterable.java |   81 -
 .../crunch/materialize/MaterializableMap.java      |   50 -
 .../materialize/pobject/CollectionPObject.java     |   55 -
 .../materialize/pobject/FirstElementPObject.java   |   50 -
 .../crunch/materialize/pobject/MapPObject.java     |   62 -
 .../crunch/materialize/pobject/PObjectImpl.java    |   85 -
 .../main/java/org/apache/crunch/package-info.java  |   25 -
 .../apache/crunch/types/CollectionDeepCopier.java  |   57 -
 .../java/org/apache/crunch/types/Converter.java    |   41 -
 .../java/org/apache/crunch/types/DeepCopier.java   |   60 -
 .../org/apache/crunch/types/MapDeepCopier.java     |   54 -
 .../org/apache/crunch/types/PGroupedTableType.java |  141 -
 .../java/org/apache/crunch/types/PTableType.java   |   44 -
 .../main/java/org/apache/crunch/types/PType.java   |   86 -
 .../java/org/apache/crunch/types/PTypeFamily.java  |   77 -
 .../java/org/apache/crunch/types/PTypeUtils.java   |   66 -
 .../main/java/org/apache/crunch/types/PTypes.java  |  252 -
 .../main/java/org/apache/crunch/types/Protos.java  |  173 -
 .../org/apache/crunch/types/TupleDeepCopier.java   |   65 -
 .../java/org/apache/crunch/types/TupleFactory.java |  134 -
 .../apache/crunch/types/avro/AvroCapabilities.java |  106 -
 .../apache/crunch/types/avro/AvroDeepCopier.java   |  209 -
 .../crunch/types/avro/AvroGroupedTableType.java    |  114 -
 .../apache/crunch/types/avro/AvroInputFormat.java  |   41 -
 .../apache/crunch/types/avro/AvroKeyConverter.java |   65 -
 .../apache/crunch/types/avro/AvroOutputFormat.java |   87 -
 .../crunch/types/avro/AvroPairConverter.java       |  108 -
 .../apache/crunch/types/avro/AvroRecordReader.java |  114 -
 .../apache/crunch/types/avro/AvroTableType.java    |  151 -
 .../crunch/types/avro/AvroTextOutputFormat.java    |   60 -
 .../org/apache/crunch/types/avro/AvroType.java     |  199 -
 .../apache/crunch/types/avro/AvroTypeFamily.java   |  164 -
 .../crunch/types/avro/AvroUtf8InputFormat.java     |   98 -
 .../java/org/apache/crunch/types/avro/Avros.java   |  709 -
 .../crunch/types/avro/ReflectDataFactory.java      |   41 -
 .../crunch/types/avro/SafeAvroSerialization.java   |  145 -
 .../org/apache/crunch/types/avro/package-info.java |   22 -
 .../java/org/apache/crunch/types/package-info.java |   22 -
 .../types/writable/GenericArrayWritable.java       |  135 -
 .../crunch/types/writable/TextMapWritable.java     |   88 -
 .../crunch/types/writable/TupleWritable.java       |  224 -
 .../crunch/types/writable/WritableDeepCopier.java  |   70 -
 .../types/writable/WritableGroupedTableType.java   |   85 -
 .../types/writable/WritablePairConverter.java      |   62 -
 .../crunch/types/writable/WritableTableType.java   |  130 -
 .../apache/crunch/types/writable/WritableType.java |  133 -
 .../crunch/types/writable/WritableTypeFamily.java  |  147 -
 .../types/writable/WritableValueConverter.java     |   60 -
 .../apache/crunch/types/writable/Writables.java    |  588 -
 .../apache/crunch/types/writable/package-info.java |   22 -
 .../java/org/apache/crunch/util/CrunchTool.java    |  118 -
 .../java/org/apache/crunch/util/DistCache.java     |  231 -
 .../org/apache/crunch/util/PartitionUtils.java     |   34 -
 .../main/java/org/apache/crunch/util/Tuples.java   |  150 -
 .../java/org/apache/crunch/util/package-info.java  |   22 -
 crunch/src/main/resources/log4j.properties         |   24 -
 crunch/src/site/site.xml                           |   34 -
 crunch/src/test/avro/employee.avsc                 |   26 -
 crunch/src/test/avro/person.avsc                   |   26 -
 .../src/test/java/org/apache/crunch/AndFnTest.java |   77 -
 .../test/java/org/apache/crunch/CombineFnTest.java |  222 -
 .../src/test/java/org/apache/crunch/NotFnTest.java |   72 -
 .../src/test/java/org/apache/crunch/OrFnTest.java  |   78 -
 .../src/test/java/org/apache/crunch/PairTest.java  |   66 -
 .../src/test/java/org/apache/crunch/TupleTest.java |  139 -
 .../test/java/org/apache/crunch/WriteModeTest.java |  103 -
 .../java/org/apache/crunch/fn/AggregatorsTest.java |  239 -
 .../org/apache/crunch/fn/ExtractKeyFnTest.java     |   44 -
 .../java/org/apache/crunch/fn/FilterFnTest.java    |   85 -
 .../java/org/apache/crunch/fn/MapKeysTest.java     |   51 -
 .../java/org/apache/crunch/fn/MapValuesTest.java   |   50 -
 .../java/org/apache/crunch/fn/PairMapTest.java     |   52 -
 .../org/apache/crunch/fn/StoreLastEmitter.java     |   41 -
 .../apache/crunch/impl/SingleUseIterableTest.java  |   54 -
 .../org/apache/crunch/impl/mr/MRPipelineTest.java  |   86 -
 .../impl/mr/collect/DoCollectionImplTest.java      |  112 -
 .../crunch/impl/mr/collect/DoTableImplTest.java    |   86 -
 .../impl/mr/emit/IntermediateEmitterTest.java      |   83 -
 .../impl/mr/exec/CappedExponentialCounterTest.java |   42 -
 .../crunch/impl/mr/exec/CrunchJobHooksTest.java    |   42 -
 .../crunch/impl/mr/plan/DotfileWriterTest.java     |  132 -
 .../crunch/impl/mr/plan/JobNameBuilderTest.java    |   41 -
 .../crunch/io/SequentialFileNamingSchemeTest.java  |   84 -
 .../apache/crunch/io/SourceTargetHelperTest.java   |   59 -
 .../crunch/io/avro/AvroFileReaderFactoryTest.java  |  184 -
 .../apache/crunch/io/avro/AvroFileSourceTest.java  |   91 -
 .../lib/AvroIndexedRecordPartitionerTest.java      |   98 -
 .../java/org/apache/crunch/lib/CartesianTest.java  |   77 -
 .../java/org/apache/crunch/lib/DistinctTest.java   |   52 -
 .../java/org/apache/crunch/lib/SampleTest.java     |   71 -
 .../org/apache/crunch/lib/SecondarySortTest.java   |   53 -
 .../crunch/lib/TupleWritablePartitionerTest.java   |   68 -
 .../lib/join/BrokenLeftAndOuterJoinTest.java       |   90 -
 .../crunch/lib/join/FullOuterJoinFnTest.java       |   48 -
 .../apache/crunch/lib/join/InnerJoinFnTest.java    |   42 -
 .../org/apache/crunch/lib/join/JoinFnTestBase.java |   82 -
 .../apache/crunch/lib/join/LeftOuterJoinTest.java  |   46 -
 .../crunch/lib/join/RightOuterJoinFnTest.java      |   46 -
 .../java/org/apache/crunch/test/CountersTest.java  |   70 -
 .../java/org/apache/crunch/test/StringWrapper.java |  102 -
 .../crunch/types/CollectionDeepCopierTest.java     |   61 -
 .../org/apache/crunch/types/MapDeepCopierTest.java |   63 -
 .../org/apache/crunch/types/PTypeUtilsTest.java    |   89 -
 .../java/org/apache/crunch/types/PTypesTest.java   |   34 -
 .../apache/crunch/types/TupleDeepCopierTest.java   |   77 -
 .../org/apache/crunch/types/TupleFactoryTest.java  |   69 -
 .../crunch/types/avro/AvroDeepCopierTest.java      |  107 -
 .../types/avro/AvroGroupedTableTypeTest.java       |   60 -
 .../crunch/types/avro/AvroTableTypeTest.java       |   72 -
 .../org/apache/crunch/types/avro/AvroTypeTest.java |  279 -
 .../org/apache/crunch/types/avro/AvrosTest.java    |  325 -
 .../types/writable/GenericArrayWritableTest.java   |   70 -
 .../types/writable/WritableDeepCopierTest.java     |   54 -
 .../writable/WritableGroupedTableTypeTest.java     |   56 -
 .../types/writable/WritableTableTypeTest.java      |   47 -
 .../crunch/types/writable/WritableTypeTest.java    |   97 -
 .../crunch/types/writable/WritablesTest.java       |  256 -
 .../java/org/apache/crunch/util/DistCacheTest.java |  156 -
 pom.xml                                            |    4 +-
 702 files changed, 70421 insertions(+), 70421 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-contrib/pom.xml b/crunch-contrib/pom.xml
index 12f9a13..e5a35c5 100644
--- a/crunch-contrib/pom.xml
+++ b/crunch-contrib/pom.xml
@@ -32,7 +32,7 @@ under the License.
   
     <dependency>
       <groupId>org.apache.crunch</groupId>
-      <artifactId>crunch</artifactId>
+      <artifactId>crunch-core</artifactId>
     </dependency>
     
     <dependency>

http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-core/pom.xml b/crunch-core/pom.xml
new file mode 100644
index 0000000..d365c3d
--- /dev/null
+++ b/crunch-core/pom.xml
@@ -0,0 +1,182 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.crunch</groupId>
+    <artifactId>crunch-parent</artifactId>
+    <version>0.6.0-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>crunch-core</artifactId>
+  <name>Apache Crunch Core</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.avro</groupId>
+      <artifactId>avro</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.avro</groupId>
+      <artifactId>avro-mapred</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.javassist</groupId>
+      <artifactId>javassist</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+    <!-- Override the slf4j dependency from Avro, which is incompatible with
+         Hadoop's. -->
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.codehaus.jackson</groupId>
+      <artifactId>jackson-core-asl</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.codehaus.jackson</groupId>
+      <artifactId>jackson-mapper-asl</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    
+    <!-- Both Protobufs and Thrift are supported as
+         derived serialization types, and you can use
+         (almost) any version of them you like, Crunch
+         only relies on the stable public APIs, not the
+         structure of the files themselves.
+
+         Both dependencies are scoped as provided, in
+         order to not expand the size of the assembly jars
+         unnecessarily.
+    -->
+
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.thrift</groupId>
+      <artifactId>libthrift</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>commons-logging</groupId>
+      <artifactId>commons-logging</artifactId>
+      <scope>provided</scope>
+    </dependency>
+   
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+    <!-- Used by LocalJobRunner in integration tests -->
+    <dependency>
+      <groupId>commons-httpclient</groupId>
+      <artifactId>commons-httpclient</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.crunch</groupId>
+      <artifactId>crunch-test</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-all</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-failsafe-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.avro</groupId>
+        <artifactId>avro-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>schemas</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>schema</goal>
+            </goals>
+            <configuration>
+              <testSourceDirectory>${project.basedir}/src/test/avro/</testSourceDirectory>
+              <testOutputDirectory>target/generated-test-sources/</testOutputDirectory>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/CancelJobsIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/CancelJobsIT.java b/crunch-core/src/it/java/org/apache/crunch/CancelJobsIT.java
new file mode 100644
index 0000000..ff01a2f
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/CancelJobsIT.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.io.To;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class CancelJobsIT {
+
+  @Rule
+  public TemporaryPath tmpDir = TemporaryPaths.create();
+
+  @Test
+  public void testRun() throws Exception {
+    PipelineExecution pe = run();
+    pe.waitUntilDone();
+    PipelineResult pr = pe.getResult();
+    assertEquals(PipelineExecution.Status.SUCCEEDED, pe.getStatus());
+    assertEquals(2, pr.getStageResults().size());
+  }
+  
+  @Test
+  public void testKill() throws Exception {
+    PipelineExecution pe = run();
+    pe.kill();
+    pe.waitUntilDone();
+    assertEquals(PipelineExecution.Status.KILLED, pe.getStatus());
+  }
+
+  @Test
+  public void testKillMultipleTimes() throws Exception {
+    PipelineExecution pe = run();
+    for (int i = 0; i < 10; i++) {
+      pe.kill();
+    }
+    pe.waitUntilDone();
+    assertEquals(PipelineExecution.Status.KILLED, pe.getStatus());
+  }
+
+  @Test
+  public void testKillAfterDone() throws Exception {
+    PipelineExecution pe = run();
+    pe.waitUntilDone();
+    assertEquals(PipelineExecution.Status.SUCCEEDED, pe.getStatus());
+    pe.kill(); // expect no-op
+    assertEquals(PipelineExecution.Status.SUCCEEDED, pe.getStatus());
+  }
+  
+  public PipelineExecution run() throws IOException {
+    String shakes = tmpDir.copyResourceFileName("shakes.txt");
+    String out = tmpDir.getFileName("cancel");
+    Pipeline p = new MRPipeline(CancelJobsIT.class, tmpDir.getDefaultConfiguration());
+    PCollection<String> words = p.readTextFile(shakes);
+    p.write(words.count().top(20), To.textFile(out));
+    return p.runAsync(); // need to hack to slow down job start up if this test becomes flaky.
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/CleanTextIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/CleanTextIT.java b/crunch-core/src/it/java/org/apache/crunch/CleanTextIT.java
new file mode 100644
index 0000000..2f4004e
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/CleanTextIT.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.io.To;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.avro.Avros;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.io.Files;
+
+/**
+ *
+ */
+public class CleanTextIT {
+
+  private static final int LINES_IN_SHAKES = 3667;
+  
+  @Rule
+  public TemporaryPath tmpDir = TemporaryPaths.create();
+  
+  static DoFn<String, String> CLEANER = new DoFn<String, String>() {
+    @Override
+    public void process(String input, Emitter<String> emitter) {
+      emitter.emit(input.toLowerCase());
+    }
+  };
+  
+  static DoFn<String, String> SPLIT = new DoFn<String, String>() {
+    @Override
+    public void process(String input, Emitter<String> emitter) {
+      for (String word : input.split("\\S+")) {
+        if (!word.isEmpty()) {
+          emitter.emit(word);
+        }
+      }
+    }
+  };
+  
+  @Test
+  public void testMapSideOutputs() throws Exception {
+    Pipeline pipeline = new MRPipeline(CleanTextIT.class, tmpDir.getDefaultConfiguration());
+    String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
+    PCollection<String> shakespeare = pipeline.readTextFile(shakesInputPath);
+    
+    PCollection<String> cleanShakes = shakespeare.parallelDo(CLEANER, Avros.strings());
+    File cso = tmpDir.getFile("cleanShakes");
+    cleanShakes.write(To.textFile(cso.getAbsolutePath()));
+    
+    File wc = tmpDir.getFile("wordCounts");
+    cleanShakes.parallelDo(SPLIT, Avros.strings()).count().write(To.textFile(wc.getAbsolutePath()));
+    pipeline.done();
+    
+    File cleanFile = new File(cso, "part-m-00000");
+    List<String> lines = Files.readLines(cleanFile, Charset.defaultCharset());
+    assertEquals(LINES_IN_SHAKES, lines.size());
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/CollectionPObjectIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/CollectionPObjectIT.java b/crunch-core/src/it/java/org/apache/crunch/CollectionPObjectIT.java
new file mode 100644
index 0000000..7e0c75c
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/CollectionPObjectIT.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.lang.String;
+import java.util.Collection;
+
+import org.apache.crunch.PCollection;
+import org.apache.crunch.PObject;
+import org.apache.crunch.Pipeline;
+import org.apache.crunch.impl.mem.MemPipeline;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.materialize.pobject.CollectionPObject;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.junit.Rule;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class CollectionPObjectIT {
+
+  private static final int LINES_IN_SHAKES = 3667;
+
+  private static final String FIRST_SHAKESPEARE_LINE =
+      "***The Project Gutenberg's Etext of Shakespeare's First Folio***";
+
+  private static final String LAST_SHAKESPEARE_LINE =
+      "FINIS. THE TRAGEDIE OF MACBETH.";
+
+  @Rule
+  public TemporaryPath tmpDir = TemporaryPaths.create();
+
+  @Test
+  public void testPObjectMRPipeline() throws IOException {
+    runPObject(new MRPipeline(CollectionPObjectIT.class, tmpDir.getDefaultConfiguration()));
+  }
+
+  @Test
+  public void testAsCollectionMRPipeline() throws IOException {
+    runAsCollection(new MRPipeline(CollectionPObjectIT.class, tmpDir.getDefaultConfiguration()));
+  }
+
+  @Test
+  public void testPObjectMemPipeline() throws IOException {
+    runPObject(MemPipeline.getInstance());
+  }
+
+  @Test
+  public void testAsCollectionMemPipeline() throws IOException {
+    runAsCollection(MemPipeline.getInstance());
+  }
+
+  private PCollection<String> getPCollection(Pipeline pipeline) throws IOException {
+    String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
+    PCollection<String> shakespeare = pipeline.readTextFile(shakesInputPath);
+    return shakespeare;
+  }
+
+  private void verifyLines(String[] lines) {
+    assertEquals("Not enough lines in Shakespeare.", LINES_IN_SHAKES, lines.length);
+    assertEquals("First line in Shakespeare is wrong.", FIRST_SHAKESPEARE_LINE, lines[0]);
+    assertEquals("Last line in Shakespeare is wrong.", LAST_SHAKESPEARE_LINE,
+        lines[lines.length - 1]);
+  }
+
+  public void runPObject(Pipeline pipeline) throws IOException {
+    PCollection<String> shakespeare = getPCollection(pipeline);
+    PObject<Collection<String>> linesP = new CollectionPObject<String>(shakespeare);
+    String[] lines = new String[LINES_IN_SHAKES];
+    lines = linesP.getValue().toArray(lines);
+    verifyLines(lines);
+  }
+
+  public void runAsCollection(Pipeline pipeline) throws IOException {
+    PCollection<String> shakespeare = getPCollection(pipeline);
+    String[] lines = new String[LINES_IN_SHAKES];
+    lines = shakespeare.asCollection().getValue().toArray(lines);
+    verifyLines(lines);
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/CollectionsIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/CollectionsIT.java b/crunch-core/src/it/java/org/apache/crunch/CollectionsIT.java
new file mode 100644
index 0000000..17d0cae
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/CollectionsIT.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.crunch.fn.Aggregators.SimpleAggregator;
+import org.apache.crunch.impl.mem.MemPipeline;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.PTypeFamily;
+import org.apache.crunch.types.avro.AvroTypeFamily;
+import org.apache.crunch.types.writable.WritableTypeFamily;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+@SuppressWarnings("serial")
+public class CollectionsIT {
+
+  private static class AggregateStringListFn extends SimpleAggregator<Collection<String>> {
+    private final Collection<String> rtn = Lists.newArrayList();
+
+    @Override
+    public void reset() {
+      rtn.clear();
+    }
+
+    @Override
+    public void update(Collection<String> values) {
+      rtn.addAll(values);
+    }
+
+    @Override
+    public Iterable<Collection<String>> results() {
+      return ImmutableList.of(rtn);
+    }
+  }
+
+  private static PTable<String, Collection<String>> listOfCharcters(PCollection<String> lines, PTypeFamily typeFamily) {
+
+    return lines.parallelDo(new DoFn<String, Pair<String, Collection<String>>>() {
+      @Override
+      public void process(String line, Emitter<Pair<String, Collection<String>>> emitter) {
+        for (String word : line.split("\\s+")) {
+          Collection<String> characters = Lists.newArrayList();
+          for (char c : word.toCharArray()) {
+            characters.add(String.valueOf(c));
+          }
+          emitter.emit(Pair.of(word, characters));
+        }
+      }
+    }, typeFamily.tableOf(typeFamily.strings(), typeFamily.collections(typeFamily.strings())))
+        .groupByKey().combineValues(new AggregateStringListFn());
+  }
+
+  @Rule
+  public TemporaryPath tmpDir = TemporaryPaths.create();
+
+  @Test
+  public void testWritables() throws IOException {
+    run(new MRPipeline(CollectionsIT.class, tmpDir.getDefaultConfiguration()), WritableTypeFamily.getInstance());
+  }
+
+  @Test
+  public void testAvro() throws IOException {
+    run(new MRPipeline(CollectionsIT.class, tmpDir.getDefaultConfiguration()), AvroTypeFamily.getInstance());
+  }
+
+  @Test
+  public void testInMemoryWritables() throws IOException {
+    run(MemPipeline.getInstance(), WritableTypeFamily.getInstance());
+  }
+
+  @Test
+  public void testInMemoryAvro() throws IOException {
+    run(MemPipeline.getInstance(), AvroTypeFamily.getInstance());
+  }
+
+  public void run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException {
+    String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
+
+    PCollection<String> shakespeare = pipeline.readTextFile(shakesInputPath);
+    Iterable<Pair<String, Collection<String>>> lines = listOfCharcters(shakespeare, typeFamily).materialize();
+
+    boolean passed = false;
+    for (Pair<String, Collection<String>> line : lines) {
+      if (line.first().startsWith("yellow")) {
+        passed = true;
+        break;
+      }
+    }
+    pipeline.done();
+    assertTrue(passed);
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/CollectionsLengthIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/CollectionsLengthIT.java b/crunch-core/src/it/java/org/apache/crunch/CollectionsLengthIT.java
new file mode 100644
index 0000000..3a38b92
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/CollectionsLengthIT.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.lang.Long;
+
+import org.apache.crunch.impl.mem.MemPipeline;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.PTypeFamily;
+import org.apache.crunch.types.avro.AvroTypeFamily;
+import org.apache.crunch.types.writable.WritableTypeFamily;
+import org.junit.Rule;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class CollectionsLengthIT {
+
+  public static final Long LINES_IN_SHAKESPEARE = 3667L;
+
+  @Rule
+  public TemporaryPath tmpDir = TemporaryPaths.create();
+
+  @Test
+  public void testWritables() throws IOException {
+    run(new MRPipeline(CollectionsIT.class, tmpDir.getDefaultConfiguration()), WritableTypeFamily.getInstance());
+  }
+
+  @Test
+  public void testAvro() throws IOException {
+    run(new MRPipeline(CollectionsIT.class, tmpDir.getDefaultConfiguration()), AvroTypeFamily.getInstance());
+  }
+
+  @Test
+  public void testInMemoryWritables() throws IOException {
+    run(MemPipeline.getInstance(), WritableTypeFamily.getInstance());
+  }
+
+  @Test
+  public void testInMemoryAvro() throws IOException {
+    run(MemPipeline.getInstance(), AvroTypeFamily.getInstance());
+  }
+
+  public void run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException {
+    String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
+
+    PCollection<String> shakespeare = pipeline.readTextFile(shakesInputPath);
+    Long length = shakespeare.length().getValue();
+    assertEquals("Incorrect length for shakespear PCollection.", LINES_IN_SHAKESPEARE, length);
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/DeepCopyCustomTuplesIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/DeepCopyCustomTuplesIT.java b/crunch-core/src/it/java/org/apache/crunch/DeepCopyCustomTuplesIT.java
new file mode 100644
index 0000000..f1323ca
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/DeepCopyCustomTuplesIT.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.apache.crunch.types.avro.Avros.*;
+import static org.junit.Assert.assertEquals;
+
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.PType;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.collect.Iterables;
+
+/**
+ *
+ */
+public class DeepCopyCustomTuplesIT {
+  @Rule
+  public TemporaryPath tmpDir = TemporaryPaths.create();
+  
+  public static class PID extends Pair<Integer, String> {
+    public PID(Integer first, String second) {
+      super(first, second);
+    }
+  }
+  
+  private static PType<PID> pids = tuples(PID.class, ints(), strings());
+  
+  @Test
+  public void testDeepCopyCustomTuple() throws Exception {
+    Pipeline p = new MRPipeline(DeepCopyCustomTuplesIT.class, tmpDir.getDefaultConfiguration());
+    String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
+    PCollection<String> shakes = p.readTextFile(shakesInputPath);
+    Iterable<String> out = shakes
+        .parallelDo(new PreProcFn(), tableOf(ints(), pairs(ints(), pids)))
+        .groupByKey()
+        .parallelDo(new PostProcFn(), strings())
+        .materialize();
+    assertEquals(65, Iterables.size(out));
+    p.done();
+  }
+  
+  private static class PreProcFn extends MapFn<String, Pair<Integer, Pair<Integer, PID>>> {
+    private int counter = 0;
+    @Override
+    public Pair<Integer, Pair<Integer, PID>> map(String input) {
+      return Pair.of(counter++, Pair.of(counter++, new PID(input.length(), input)));
+    }
+  };
+  
+  private static class PostProcFn extends DoFn<Pair<Integer, Iterable<Pair<Integer, PID>>>, String> {
+    @Override
+    public void process(Pair<Integer, Iterable<Pair<Integer, PID>>> input, Emitter<String> emitter) {
+      for (Pair<Integer, PID> p : input.second()) {
+        if (p.second().first() > 0 && p.second().first() < 10) {
+          emitter.emit(p.second().second());
+        }
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/EnumPairIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/EnumPairIT.java b/crunch-core/src/it/java/org/apache/crunch/EnumPairIT.java
new file mode 100644
index 0000000..1d0974e
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/EnumPairIT.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.PTypes;
+import org.apache.crunch.types.writable.Writables;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class EnumPairIT implements Serializable {
+  @Rule
+  public transient TemporaryPath tmpDir = TemporaryPaths.create();
+
+  static enum etypes {
+    type1,
+  }
+
+  @Test
+  public void testEnumPTypes() throws IOException {
+    String inputFile1 = tmpDir.copyResourceFileName("set1.txt");
+    Pipeline pipeline = new MRPipeline(EnumPairIT.class);
+    PCollection<String> set1 = pipeline.readTextFile(inputFile1);
+    PTable<String, etypes> data = set1.parallelDo(new DoFn<String, Pair<String, etypes>>() {
+      @Override
+      public void process(String input, Emitter<Pair<String, etypes>> emitter) {
+        emitter.emit(new Pair<String, etypes>(input, etypes.type1));
+      }
+    }, Writables.tableOf(Writables.strings(), PTypes.enums(etypes.class, set1.getTypeFamily())));
+
+    Iterable<Pair<String, etypes>> materialized = data.materialize();
+    pipeline.run();
+    for (Pair<String, etypes> pair : materialized) {
+      assertEquals(etypes.type1, pair.second());
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/FirstElementPObjectIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/FirstElementPObjectIT.java b/crunch-core/src/it/java/org/apache/crunch/FirstElementPObjectIT.java
new file mode 100644
index 0000000..d985e10
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/FirstElementPObjectIT.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.lang.String;
+
+import org.apache.crunch.PCollection;
+import org.apache.crunch.PObject;
+import org.apache.crunch.impl.mem.MemPipeline;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.materialize.pobject.FirstElementPObject;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.junit.Rule;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class FirstElementPObjectIT {
+
+  private static final String FIRST_SHAKESPEARE_LINE =
+      "***The Project Gutenberg's Etext of Shakespeare's First Folio***";
+
+  @Rule
+  public TemporaryPath tmpDir = TemporaryPaths.create();
+
+  @Test
+  public void testMRPipeline() throws IOException {
+    run(new MRPipeline(FirstElementPObjectIT.class, tmpDir.getDefaultConfiguration()));
+  }
+
+  @Test
+  public void testInMemoryPipeline() throws IOException {
+    run(MemPipeline.getInstance());
+  }
+
+  public void run(Pipeline pipeline) throws IOException {
+    String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
+    PCollection<String> shakespeare = pipeline.readTextFile(shakesInputPath);
+    PObject<String> firstLine = new FirstElementPObject<String>(shakespeare);
+    String first = firstLine.getValue();
+    assertEquals("First line in Shakespeare is wrong.", FIRST_SHAKESPEARE_LINE, first);
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/IterableReuseProtectionIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/IterableReuseProtectionIT.java b/crunch-core/src/it/java/org/apache/crunch/IterableReuseProtectionIT.java
new file mode 100644
index 0000000..da487eb
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/IterableReuseProtectionIT.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.crunch.fn.IdentityFn;
+import org.apache.crunch.impl.mem.MemPipeline;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.writable.Writables;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Verify that calling the iterator method on a Reducer-based Iterable 
+ * is forcefully disallowed.
+ */
+public class IterableReuseProtectionIT {
+
+  @Rule
+  public TemporaryPath tmpDir = TemporaryPaths.create();
+  
+  
+  public void checkIteratorReuse(Pipeline pipeline) throws IOException {
+    Iterable<String> values = pipeline.readTextFile(tmpDir.copyResourceFileName("set1.txt"))
+        .by(IdentityFn.<String>getInstance(), Writables.strings())
+        .groupByKey()
+        .combineValues(new TestIterableReuseFn())
+        .values().materialize();
+    
+    List<String> valueList = Lists.newArrayList(values);
+    Collections.sort(valueList);
+    assertEquals(Lists.newArrayList("a", "b", "c", "e"), valueList);
+  }
+  
+  @Test
+  public void testIteratorReuse_MRPipeline() throws IOException {
+    checkIteratorReuse(new MRPipeline(IterableReuseProtectionIT.class, tmpDir.getDefaultConfiguration()));
+  }
+  
+  @Test
+  public void testIteratorReuse_InMemoryPipeline() throws IOException {
+    checkIteratorReuse(MemPipeline.getInstance());
+  }
+  
+  static class TestIterableReuseFn extends CombineFn<String, String> {
+
+    @Override
+    public void process(Pair<String, Iterable<String>> input, Emitter<Pair<String, String>> emitter) {
+      StringBuilder combinedBuilder = new StringBuilder();
+      for (String v : input.second()) {
+        combinedBuilder.append(v);
+      }
+      
+      try {
+        input.second().iterator();
+        throw new RuntimeException("Second call to iterator should throw an exception");
+      } catch (IllegalStateException e) {
+        // Expected situation
+      }
+      emitter.emit(Pair.of(input.first(), combinedBuilder.toString()));
+    }
+    
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/MRPipelineIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/MRPipelineIT.java b/crunch-core/src/it/java/org/apache/crunch/MRPipelineIT.java
new file mode 100644
index 0000000..7670e88
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/MRPipelineIT.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.crunch.fn.FilterFns;
+import org.apache.crunch.fn.IdentityFn;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.io.To;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.writable.Writables;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class MRPipelineIT implements Serializable {
+  @Rule
+  public transient TemporaryPath tmpDir = TemporaryPaths.create();
+
+  @Test
+  public void materializedColShouldBeWritten() throws Exception {
+    File textFile = tmpDir.copyResourceFile("shakes.txt");
+    Pipeline pipeline = new MRPipeline(MRPipelineIT.class, tmpDir.getDefaultConfiguration());
+    PCollection<String> genericCollection = pipeline.readTextFile(textFile.getAbsolutePath());
+    pipeline.run();
+    PCollection<String> filter = genericCollection.filter("Filtering data", FilterFns.<String>ACCEPT_ALL());
+    filter.materialize();
+    pipeline.run();
+    File file = tmpDir.getFile("output.txt");
+    Target outFile = To.textFile(file.getAbsolutePath());
+    PCollection<String> write = filter.write(outFile);
+    write.materialize();
+    pipeline.run();
+  }
+  
+  
+  
+  @Test
+  public void testPGroupedTableToMultipleOutputs() throws IOException{
+    Pipeline pipeline = new MRPipeline(MRPipelineIT.class, tmpDir.getDefaultConfiguration());
+    PGroupedTable<String, String> groupedLineTable = pipeline.readTextFile(tmpDir.copyResourceFileName("set1.txt")).by(IdentityFn.<String>getInstance(), Writables.strings()).groupByKey();
+    
+    PTable<String, String> ungroupedTableA = groupedLineTable.ungroup();
+    PTable<String, String> ungroupedTableB = groupedLineTable.ungroup();
+    
+    File outputDirA = tmpDir.getFile("output_a");
+    File outputDirB = tmpDir.getFile("output_b");
+    
+    pipeline.writeTextFile(ungroupedTableA, outputDirA.getAbsolutePath());
+    pipeline.writeTextFile(ungroupedTableB, outputDirB.getAbsolutePath());
+    pipeline.done();
+
+    // Verify that output from a single PGroupedTable can be sent to multiple collections
+    assertTrue(new File(outputDirA, "part-r-00000").exists());
+    assertTrue(new File(outputDirB, "part-r-00000").exists());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch-core/src/it/java/org/apache/crunch/MapPObjectIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/MapPObjectIT.java b/crunch-core/src/it/java/org/apache/crunch/MapPObjectIT.java
new file mode 100644
index 0000000..c48284f
--- /dev/null
+++ b/crunch-core/src/it/java/org/apache/crunch/MapPObjectIT.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch;
+
+import static junit.framework.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.crunch.impl.mem.MemPipeline;
+import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.materialize.pobject.MapPObject;
+import org.apache.crunch.test.TemporaryPath;
+import org.apache.crunch.test.TemporaryPaths;
+import org.apache.crunch.types.PTypeFamily;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableList;
+
+public class MapPObjectIT {
+
+  static final ImmutableList<Pair<Integer, String>> kvPairs = ImmutableList.of(Pair.of(0, "a"), Pair.of(1, "b"),
+      Pair.of(2, "c"), Pair.of(3, "e"));
+
+  public void assertMatches(Map<Integer, String> m) {
+    for (Integer k : m.keySet()) {
+      assertEquals(kvPairs.get(k).second(), m.get(k));
+    }
+  }
+
+  private static class Set1Mapper extends MapFn<String, Pair<Integer, String>> {
+    @Override
+    public Pair<Integer, String> map(String input) {
+
+      int k = -1;
+      if (input.equals("a"))
+        k = 0;
+      else if (input.equals("b"))
+        k = 1;
+      else if (input.equals("c"))
+        k = 2;
+      else if (input.equals("e"))
+        k = 3;
+      return Pair.of(k, input);
+    }
+  }
+  @Rule
+  public TemporaryPath tmpDir = TemporaryPaths.create();
+
+  @Test
+  public void testMemMapPObject() {
+    PTable<Integer, String> table = MemPipeline.tableOf(kvPairs);
+    PObject<Map<Integer, String>> map = new MapPObject<Integer, String>(table);
+    assertMatches(map.getValue());
+  }
+
+  @Test
+  public void testMemAsMap() {
+    PTable<Integer, String> table = MemPipeline.tableOf(kvPairs);
+    assertMatches(table.asMap().getValue());
+  }
+
+  private PTable<Integer, String> getMRPTable() throws IOException {
+    Pipeline p = new MRPipeline(MaterializeToMapIT.class, tmpDir.getDefaultConfiguration());
+    String inputFile = tmpDir.copyResourceFileName("set1.txt");
+    PCollection<String> c = p.readTextFile(inputFile);
+    PTypeFamily tf = c.getTypeFamily();
+    PTable<Integer, String> table = c.parallelDo(new Set1Mapper(), tf.tableOf(tf.ints(),
+        tf.strings()));
+    return table;
+  }
+
+  @Test
+  public void testMRMapPObject() throws IOException {
+    PTable<Integer, String> table = getMRPTable();
+    PObject<Map<Integer, String>> map = new MapPObject<Integer, String>(table);
+    assertMatches(map.getValue());
+  }
+
+  @Test
+  public void testMRAsMap() throws IOException {
+    PTable<Integer, String> table = getMRPTable();
+    assertMatches(table.asMap().getValue());
+  }
+}


Mime
View raw message