Return-Path: X-Original-To: apmail-crunch-commits-archive@www.apache.org Delivered-To: apmail-crunch-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CAE5C10A98 for ; Tue, 15 Oct 2013 04:21:39 +0000 (UTC) Received: (qmail 57247 invoked by uid 500); 15 Oct 2013 04:21:39 -0000 Delivered-To: apmail-crunch-commits-archive@crunch.apache.org Received: (qmail 57000 invoked by uid 500); 15 Oct 2013 04:21:35 -0000 Mailing-List: contact commits-help@crunch.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@crunch.apache.org Delivered-To: mailing list commits@crunch.apache.org Received: (qmail 56987 invoked by uid 99); 15 Oct 2013 04:21:34 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Oct 2013 04:21:34 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 1407A8B40A6; Tue, 15 Oct 2013 04:21:33 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jwills@apache.org To: commits@crunch.apache.org Date: Tue, 15 Oct 2013 04:21:34 -0000 Message-Id: <0484575f903d41f89b06d85d2d792cbf@git.apache.org> In-Reply-To: <0d41c37b01204a918ed9297d6700b3c6@git.apache.org> References: <0d41c37b01204a918ed9297d6700b3c6@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/2] git commit: CRUNCH-276: Various static checks and FindBugs fixes. Contributed by Sean Owen. CRUNCH-276: Various static checks and FindBugs fixes. Contributed by Sean Owen. Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/e0dbebf1 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/e0dbebf1 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/e0dbebf1 Branch: refs/heads/master Commit: e0dbebf1546973f9994e0d42438e94ecd2088d00 Parents: 68cf3bd Author: Josh Wills Authored: Mon Oct 14 21:16:57 2013 -0700 Committer: Josh Wills Committed: Mon Oct 14 21:16:57 2013 -0700 ---------------------------------------------------------------------- .../contrib/bloomfilter/BloomFilterFactory.java | 43 +++--- .../contrib/bloomfilter/BloomFilterFn.java | 3 +- .../contrib/text/AbstractSimpleExtractor.java | 4 +- .../apache/crunch/contrib/text/Extractors.java | 67 +++++---- .../apache/crunch/contrib/text/ParseTest.java | 6 +- crunch-core/pom.xml | 1 - .../it/java/org/apache/crunch/CheckpointIT.java | 3 +- .../org/apache/crunch/CollectionsLengthIT.java | 1 - .../it/java/org/apache/crunch/EnumPairIT.java | 2 +- .../apache/crunch/LongPipelinePlannerIT.java | 9 +- .../org/apache/crunch/MaterializeToMapIT.java | 17 +-- .../crunch/SingleUseIterableExceptionIT.java | 5 +- .../java/org/apache/crunch/TermFrequencyIT.java | 9 +- .../it/java/org/apache/crunch/WordCountIT.java | 5 +- .../org/apache/crunch/lib/AvroTypeSortIT.java | 9 +- .../crunch/lib/SpecificAvroGroupByIT.java | 8 +- .../crunch/lib/join/MapsideJoinStrategyIT.java | 3 +- .../java/org/apache/crunch/PipelineResult.java | 2 +- .../src/main/java/org/apache/crunch/TupleN.java | 13 +- .../crunch/impl/mem/collect/MemTable.java | 1 - .../impl/mr/collect/PGroupedTableImpl.java | 2 +- .../impl/mr/emit/IntermediateEmitter.java | 5 +- .../org/apache/crunch/impl/mr/plan/DoNode.java | 2 +- .../org/apache/crunch/impl/mr/plan/Edge.java | 7 +- .../crunch/impl/mr/plan/PlanningParameters.java | 2 +- .../crunch/impl/mr/run/RuntimeParameters.java | 2 +- .../crunch/io/avro/trevni/TrevniKeySource.java | 3 +- .../crunch/io/avro/trevni/TrevniKeyTarget.java | 2 +- .../io/avro/trevni/TrevniOutputFormat.java | 5 - .../crunch/io/impl/AutoClosingIterator.java | 6 +- .../java/org/apache/crunch/lib/Cogroup.java | 6 +- .../java/org/apache/crunch/lib/Distinct.java | 2 +- .../main/java/org/apache/crunch/lib/Sample.java | 3 +- .../java/org/apache/crunch/lib/SampleUtils.java | 32 +++-- .../main/java/org/apache/crunch/lib/Sort.java | 14 +- .../lib/join/BloomFilterJoinStrategy.java | 20 ++- .../org/apache/crunch/lib/join/InnerJoinFn.java | 6 +- .../org/apache/crunch/lib/join/JoinUtils.java | 3 +- .../crunch/lib/sort/ReverseAvroComparator.java | 4 +- .../org/apache/crunch/lib/sort/SortFns.java | 2 +- .../materialize/MaterializableIterable.java | 3 +- .../org/apache/crunch/types/PTypeUtils.java | 2 +- .../java/org/apache/crunch/types/PTypes.java | 30 ++-- .../crunch/types/avro/AvroTextOutputFormat.java | 2 +- .../writable/WritableGroupedTableType.java | 2 +- .../crunch/impl/mr/plan/JobNameBuilderTest.java | 10 +- .../io/avro/AvroFileReaderFactoryTest.java | 5 +- .../apache/crunch/types/TupleFactoryTest.java | 1 + .../AvroSpecificDeepCopierClassloaderTest.java | 2 +- .../org/apache/crunch/examples/SortExample.java | 4 - .../crunch/examples/WordAggregationHBase.java | 44 +++--- .../apache/crunch/io/hbase/HFileTargetIT.java | 24 ++-- .../crunch/io/hbase/WordCountHBaseIT.java | 139 +++++++++---------- .../io/hbase/HFileOutputFormatForCrunch.java | 3 + .../crunch/scrunch/ScalaReflectDataFactory.java | 2 + .../scrunch/ScalaSafeReflectDatumReader.java | 19 +-- .../org/apache/crunch/test/TemporaryPath.java | 4 +- 57 files changed, 309 insertions(+), 326 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-contrib/src/main/java/org/apache/crunch/contrib/bloomfilter/BloomFilterFactory.java ---------------------------------------------------------------------- diff --git a/crunch-contrib/src/main/java/org/apache/crunch/contrib/bloomfilter/BloomFilterFactory.java b/crunch-contrib/src/main/java/org/apache/crunch/contrib/bloomfilter/BloomFilterFactory.java index 9191a6c..f7c2ebe 100644 --- a/crunch-contrib/src/main/java/org/apache/crunch/contrib/bloomfilter/BloomFilterFactory.java +++ b/crunch-contrib/src/main/java/org/apache/crunch/contrib/bloomfilter/BloomFilterFactory.java @@ -77,32 +77,33 @@ public class BloomFilterFactory { return table.groupByKey(1).combineValues(new BloomFilterAggregator()); } -} -@SuppressWarnings("serial") -class BloomFilterAggregator implements Aggregator { - private transient BloomFilter bloomFilter = null; - private transient int filterSize; + @SuppressWarnings("serial") + private static class BloomFilterAggregator implements Aggregator { + private transient BloomFilter bloomFilter = null; + private transient int filterSize; - @Override - public void update(BloomFilter value) { - bloomFilter.or(value); - } + @Override + public void update(BloomFilter value) { + bloomFilter.or(value); + } - @Override - public Iterable results() { - return ImmutableList.of(bloomFilter); - } + @Override + public Iterable results() { + return ImmutableList.of(bloomFilter); + } - @Override - public void initialize(Configuration configuration) { - filterSize = BloomFilterFn.getBloomFilterSize(configuration); - } + @Override + public void initialize(Configuration configuration) { + filterSize = BloomFilterFn.getBloomFilterSize(configuration); + } - @Override - public void reset() { - bloomFilter = BloomFilterFn.initializeFilter(filterSize); + @Override + public void reset() { + bloomFilter = BloomFilterFn.initializeFilter(filterSize); + + } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-contrib/src/main/java/org/apache/crunch/contrib/bloomfilter/BloomFilterFn.java ---------------------------------------------------------------------- diff --git a/crunch-contrib/src/main/java/org/apache/crunch/contrib/bloomfilter/BloomFilterFn.java b/crunch-contrib/src/main/java/org/apache/crunch/contrib/bloomfilter/BloomFilterFn.java index 7d27b33..c12a610 100644 --- a/crunch-contrib/src/main/java/org/apache/crunch/contrib/bloomfilter/BloomFilterFn.java +++ b/crunch-contrib/src/main/java/org/apache/crunch/contrib/bloomfilter/BloomFilterFn.java @@ -46,8 +46,9 @@ public abstract class BloomFilterFn extends DoFn @Override public void process(S input, Emitter> emitter) { Collection keys = generateKeys(input); - if (CollectionUtils.isNotEmpty(keys)) + if (CollectionUtils.isNotEmpty(keys)) { bloomFilter.add(keys); + } } public abstract Collection generateKeys(S input); http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-contrib/src/main/java/org/apache/crunch/contrib/text/AbstractSimpleExtractor.java ---------------------------------------------------------------------- diff --git a/crunch-contrib/src/main/java/org/apache/crunch/contrib/text/AbstractSimpleExtractor.java b/crunch-contrib/src/main/java/org/apache/crunch/contrib/text/AbstractSimpleExtractor.java index b0dc494..aebc60a 100644 --- a/crunch-contrib/src/main/java/org/apache/crunch/contrib/text/AbstractSimpleExtractor.java +++ b/crunch-contrib/src/main/java/org/apache/crunch/contrib/text/AbstractSimpleExtractor.java @@ -35,11 +35,11 @@ public abstract class AbstractSimpleExtractor implements Extractor { private final T defaultValue; private final TokenizerFactory scannerFactory; - public AbstractSimpleExtractor(T defaultValue) { + protected AbstractSimpleExtractor(T defaultValue) { this(defaultValue, TokenizerFactory.getDefaultInstance()); } - public AbstractSimpleExtractor(T defaultValue, TokenizerFactory scannerFactory) { + protected AbstractSimpleExtractor(T defaultValue, TokenizerFactory scannerFactory) { this.defaultValue = defaultValue; this.scannerFactory = scannerFactory; } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-contrib/src/main/java/org/apache/crunch/contrib/text/Extractors.java ---------------------------------------------------------------------- diff --git a/crunch-contrib/src/main/java/org/apache/crunch/contrib/text/Extractors.java b/crunch-contrib/src/main/java/org/apache/crunch/contrib/text/Extractors.java index 0ed1282..3ee4667 100644 --- a/crunch-contrib/src/main/java/org/apache/crunch/contrib/text/Extractors.java +++ b/crunch-contrib/src/main/java/org/apache/crunch/contrib/text/Extractors.java @@ -19,7 +19,6 @@ package org.apache.crunch.contrib.text; import java.lang.reflect.Constructor; import java.util.Collection; -import java.util.Scanner; import org.apache.crunch.Pair; import org.apache.crunch.Tuple; @@ -37,7 +36,7 @@ import com.google.common.collect.Lists; /** * Factory methods for constructing common {@code Extractor} types. */ -public class Extractors { +public final class Extractors { /** * Returns an Extractor for integers. @@ -71,7 +70,7 @@ public class Extractors { * Returns an Extractor for floats. */ public static Extractor xfloat() { - return xfloat(0f); + return xfloat(0.0f); } public static Extractor xfloat(Float defaultValue) { @@ -161,7 +160,7 @@ public class Extractors { private static class IntExtractor extends AbstractSimpleExtractor { - public IntExtractor(Integer defaultValue) { + IntExtractor(Integer defaultValue) { super(defaultValue); } @@ -182,7 +181,7 @@ public class Extractors { } private static class LongExtractor extends AbstractSimpleExtractor { - public LongExtractor(Long defaultValue) { + LongExtractor(Long defaultValue) { super(defaultValue); } @@ -200,10 +199,10 @@ public class Extractors { public String toString() { return "xlong"; } - }; - + } + private static class FloatExtractor extends AbstractSimpleExtractor { - public FloatExtractor(Float defaultValue) { + FloatExtractor(Float defaultValue) { super(defaultValue); } @@ -221,10 +220,10 @@ public class Extractors { public String toString() { return "xfloat"; } - }; - + } + private static class DoubleExtractor extends AbstractSimpleExtractor { - public DoubleExtractor(Double defaultValue) { + DoubleExtractor(Double defaultValue) { super(defaultValue); } @@ -242,11 +241,11 @@ public class Extractors { public String toString() { return "xdouble"; } - }; - + } + private static class BooleanExtractor extends AbstractSimpleExtractor { - public BooleanExtractor(Boolean defaultValue) { + BooleanExtractor(Boolean defaultValue) { super(defaultValue); } @@ -264,11 +263,11 @@ public class Extractors { public String toString() { return "xboolean"; } - }; - + } + private static class StringExtractor extends AbstractSimpleExtractor { - public StringExtractor(String defaultValue) { + StringExtractor(String defaultValue) { super(defaultValue); } @@ -286,8 +285,8 @@ public class Extractors { public String toString() { return "xstring"; } - }; - + } + private static class CollectionExtractor implements Extractor> { private final TokenizerFactory tokenizerFactory; @@ -295,7 +294,7 @@ public class Extractors { private int errors = 0; private boolean errorOnLast; - public CollectionExtractor(TokenizerFactory tokenizerFactory, Extractor extractor) { + CollectionExtractor(TokenizerFactory tokenizerFactory, Extractor extractor) { this.tokenizerFactory = tokenizerFactory; this.extractor = extractor; } @@ -322,7 +321,7 @@ public class Extractors { @Override public Collection getDefaultValue() { - return ImmutableList.of(); + return ImmutableList.of(); } @Override @@ -349,7 +348,7 @@ public class Extractors { private final Extractor one; private final Extractor two; - public PairExtractor(TokenizerFactory scannerFactory, Extractor one, Extractor two) { + PairExtractor(TokenizerFactory scannerFactory, Extractor one, Extractor two) { super(scannerFactory, ImmutableList.>of(one, two)); this.one = one; this.two = two; @@ -374,14 +373,14 @@ public class Extractors { public Pair getDefaultValue() { return Pair.of(one.getDefaultValue(), two.getDefaultValue()); } - }; - + } + private static class TripExtractor extends AbstractCompositeExtractor> { private final Extractor one; private final Extractor two; private final Extractor three; - public TripExtractor(TokenizerFactory sf, Extractor one, Extractor two, Extractor three) { + TripExtractor(TokenizerFactory sf, Extractor one, Extractor two, Extractor three) { super(sf, ImmutableList.>of(one, two, three)); this.one = one; this.two = two; @@ -407,15 +406,15 @@ public class Extractors { public String toString() { return "xtriple(" + one + "," + two + "," + three + ")"; } - }; - + } + private static class QuadExtractor extends AbstractCompositeExtractor> { private final Extractor one; private final Extractor two; private final Extractor three; private final Extractor four; - public QuadExtractor(TokenizerFactory sf, Extractor one, Extractor two, Extractor three, + QuadExtractor(TokenizerFactory sf, Extractor one, Extractor two, Extractor three, Extractor four) { super(sf, ImmutableList.>of(one, two, three, four)); this.one = one; @@ -445,12 +444,12 @@ public class Extractors { public String toString() { return "xquad(" + one + "," + two + "," + three + "," + four + ")"; } - }; - + } + private static class TupleNExtractor extends AbstractCompositeExtractor { private final Extractor[] extractors; - public TupleNExtractor(TokenizerFactory scannerFactory, Extractor...extractors) { + TupleNExtractor(TokenizerFactory scannerFactory, Extractor...extractors) { super(scannerFactory, ImmutableList.>copyOf(extractors)); this.extractors = extractors; } @@ -482,8 +481,8 @@ public class Extractors { public String toString() { return "xtupleN(" + Joiner.on(',').join(extractors) + ")"; } - }; - + } + private static class CustomTupleExtractor extends AbstractCompositeExtractor { private final Class clazz; @@ -491,7 +490,7 @@ public class Extractors { private transient Constructor constructor; - public CustomTupleExtractor(TokenizerFactory sf, Class clazz, Extractor... extractors) { + CustomTupleExtractor(TokenizerFactory sf, Class clazz, Extractor... extractors) { super(sf, ImmutableList.>copyOf(extractors)); this.clazz = clazz; this.extractors = extractors; http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-contrib/src/test/java/org/apache/crunch/contrib/text/ParseTest.java ---------------------------------------------------------------------- diff --git a/crunch-contrib/src/test/java/org/apache/crunch/contrib/text/ParseTest.java b/crunch-contrib/src/test/java/org/apache/crunch/contrib/text/ParseTest.java index 4da7521..a0e8471 100644 --- a/crunch-contrib/src/test/java/org/apache/crunch/contrib/text/ParseTest.java +++ b/crunch-contrib/src/test/java/org/apache/crunch/contrib/text/ParseTest.java @@ -29,16 +29,12 @@ import org.apache.crunch.Pair; import org.apache.crunch.Tuple3; import org.apache.crunch.Tuple4; import org.apache.crunch.TupleN; -import org.apache.crunch.contrib.text.Parse; import org.apache.crunch.impl.mem.MemPipeline; import org.apache.crunch.types.avro.Avros; import org.junit.Test; import com.google.common.collect.ImmutableList; -/** - * - */ public class ParseTest { @Test @@ -105,7 +101,7 @@ public class ParseTest { TokenizerFactory inner = TokenizerFactory.builder().delimiter(",").build(); Extractor, Tuple3>> extractor = xpair(outer, xpair(inner, xlong(), xint()), xtriple(inner, xstring(), xint(), xfloat())); - assertEquals(Pair.of(Pair.of(1L, 2), Tuple3.of("a", 17, 29f)), + assertEquals(Pair.of(Pair.of(1L, 2), Tuple3.of("a", 17, 29.0f)), extractor.extract("1,2;a,17,29")); } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/pom.xml ---------------------------------------------------------------------- diff --git a/crunch-core/pom.xml b/crunch-core/pom.xml index 8a5beb6..9530a91 100644 --- a/crunch-core/pom.xml +++ b/crunch-core/pom.xml @@ -121,7 +121,6 @@ under the License. commons-logging commons-logging - provided http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/it/java/org/apache/crunch/CheckpointIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/CheckpointIT.java b/crunch-core/src/it/java/org/apache/crunch/CheckpointIT.java index acb039d..facbc72 100644 --- a/crunch-core/src/it/java/org/apache/crunch/CheckpointIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/CheckpointIT.java @@ -78,8 +78,7 @@ public class CheckpointIT { public static PipelineResult run(Pipeline pipeline, TemporaryPath tmpDir, String shakesInputPath, String intermediatePath, - final boolean fail) - throws Exception { + final boolean fail) { PCollection shakes = pipeline.readTextFile(shakesInputPath); PTable cnts = shakes.parallelDo("split words", new DoFn() { @Override http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/it/java/org/apache/crunch/CollectionsLengthIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/CollectionsLengthIT.java b/crunch-core/src/it/java/org/apache/crunch/CollectionsLengthIT.java index 3a38b92..f1a33a2 100644 --- a/crunch-core/src/it/java/org/apache/crunch/CollectionsLengthIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/CollectionsLengthIT.java @@ -20,7 +20,6 @@ package org.apache.crunch; import static org.junit.Assert.assertEquals; import java.io.IOException; -import java.lang.Long; import org.apache.crunch.impl.mem.MemPipeline; import org.apache.crunch.impl.mr.MRPipeline; http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/it/java/org/apache/crunch/EnumPairIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/EnumPairIT.java b/crunch-core/src/it/java/org/apache/crunch/EnumPairIT.java index 1d0974e..c73a45c 100644 --- a/crunch-core/src/it/java/org/apache/crunch/EnumPairIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/EnumPairIT.java @@ -34,7 +34,7 @@ public class EnumPairIT implements Serializable { @Rule public transient TemporaryPath tmpDir = TemporaryPaths.create(); - static enum etypes { + enum etypes { type1, } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/it/java/org/apache/crunch/LongPipelinePlannerIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/LongPipelinePlannerIT.java b/crunch-core/src/it/java/org/apache/crunch/LongPipelinePlannerIT.java index d7a4b4d..2cd63f2 100644 --- a/crunch-core/src/it/java/org/apache/crunch/LongPipelinePlannerIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/LongPipelinePlannerIT.java @@ -26,6 +26,8 @@ import org.apache.crunch.test.TemporaryPaths; import org.junit.Rule; import org.junit.Test; +import java.util.Locale; + /** * Verifies that complex plans execute dependent jobs in the correct sequence. */ @@ -41,12 +43,12 @@ public class LongPipelinePlannerIT { tmpDir.getFileName("output")); } - public static void run(Pipeline p, String input, String output) throws Exception { + public static void run(Pipeline p, String input, String output) { PCollection in = p.readTextFile(input); PCollection toLower = in.parallelDo("tolower", new DoFn() { @Override public void process(String input, Emitter emitter) { - emitter.emit(input.toLowerCase()); + emitter.emit(input.toLowerCase(Locale.ENGLISH)); } }, strings()); @@ -68,7 +70,6 @@ public class LongPipelinePlannerIT { MaterializableIterable matIt = (MaterializableIterable)iso.materialize(); ParallelDoOptions.Builder builder = ParallelDoOptions.builder().sourceTargets((SourceTarget)matIt.getSource()); - final String collectionPath = matIt.getPath().toString(); PTable splitMap = keyedLower.parallelDo("split-map", new MapFn, Pair>() { @@ -100,7 +101,7 @@ public class LongPipelinePlannerIT { PCollection upper = merged.parallelDo("toupper", new MapFn() { @Override public String map(String input) { - return input.toUpperCase(); + return input.toUpperCase(Locale.ENGLISH); } }, strings()); http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java b/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java index 7fef30e..8457bac 100644 --- a/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/MaterializeToMapIT.java @@ -17,7 +17,7 @@ */ package org.apache.crunch; -import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertEquals; import java.io.IOException; import java.util.Map; @@ -37,9 +37,9 @@ public class MaterializeToMapIT { static final ImmutableList> kvPairs = ImmutableList.of(Pair.of(0, "a"), Pair.of(1, "b"), Pair.of(2, "c"), Pair.of(3, "e")); - public void assertMatches(Map m) { - for (Integer k : m.keySet()) { - assertEquals(kvPairs.get(k).second(), m.get(k)); + public static void assertMatches(Map m) { + for (Map.Entry entry : m.entrySet()) { + assertEquals(kvPairs.get(entry.getKey()).second(), entry.getValue()); } } @@ -53,14 +53,15 @@ public class MaterializeToMapIT { public Pair map(String input) { int k = -1; - if (input.equals("a")) + if (input.equals("a")) { k = 0; - else if (input.equals("b")) + } else if (input.equals("b")) { k = 1; - else if (input.equals("c")) + } else if (input.equals("c")) { k = 2; - else if (input.equals("e")) + } else if (input.equals("e")) { k = 3; + } return Pair.of(k, input); } } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/it/java/org/apache/crunch/SingleUseIterableExceptionIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/SingleUseIterableExceptionIT.java b/crunch-core/src/it/java/org/apache/crunch/SingleUseIterableExceptionIT.java index ccc91c6..ff2897b 100644 --- a/crunch-core/src/it/java/org/apache/crunch/SingleUseIterableExceptionIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/SingleUseIterableExceptionIT.java @@ -27,13 +27,10 @@ import org.apache.crunch.types.avro.Avros; import org.junit.Rule; import org.junit.Test; -/** - * - */ public class SingleUseIterableExceptionIT { @Rule - public transient TemporaryPath tmpDir = TemporaryPaths.create(); + public TemporaryPath tmpDir = TemporaryPaths.create(); static class ReduceFn extends MapFn, String> { @Override http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/it/java/org/apache/crunch/TermFrequencyIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/TermFrequencyIT.java b/crunch-core/src/it/java/org/apache/crunch/TermFrequencyIT.java index ca66aa8..2abdb8c 100644 --- a/crunch-core/src/it/java/org/apache/crunch/TermFrequencyIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/TermFrequencyIT.java @@ -23,11 +23,12 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; import java.io.Serializable; +import java.util.Locale; import org.apache.crunch.impl.mem.MemPipeline; import org.apache.crunch.impl.mr.MRPipeline; import org.apache.crunch.io.At; -import org.apache.crunch.io.ReadableSourceTarget; +import org.apache.crunch.io.ReadableSource; import org.apache.crunch.lib.Aggregate; import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.test.TemporaryPaths; @@ -85,8 +86,8 @@ public class TermFrequencyIT implements Serializable { String title = kv[0]; String text = kv[1]; for (String word : text.split("\\W+")) { - if (word.length() > 0) { - Pair pair = Pair.of(word.toLowerCase(), title); + if (!word.isEmpty()) { + Pair pair = Pair.of(word.toLowerCase(Locale.ENGLISH), title); emitter.emit(pair); } } @@ -119,7 +120,7 @@ public class TermFrequencyIT implements Serializable { pipeline.run(); // test the case we should see - Iterable lines = ((ReadableSourceTarget) st).read(pipeline.getConfiguration()); + Iterable lines = ((ReadableSource) st).read(pipeline.getConfiguration()); boolean passed = false; for (String line : lines) { if ("[well,A]\t0".equals(line)) { http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/it/java/org/apache/crunch/WordCountIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/WordCountIT.java b/crunch-core/src/it/java/org/apache/crunch/WordCountIT.java index c646663..cee6a90 100644 --- a/crunch-core/src/it/java/org/apache/crunch/WordCountIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/WordCountIT.java @@ -48,7 +48,7 @@ public class WordCountIT { enum WordCountStats { ANDS - }; + } public static PTable wordCount(PCollection words, PTypeFamily typeFamily) { return Aggregate.count(words.parallelDo(new DoFn() { @@ -68,8 +68,9 @@ public class WordCountIT { public static PTable substr(PTable ptable) { return ptable.parallelDo(new DoFn, Pair>() { + @Override public void process(Pair input, Emitter> emitter) { - if (input.first().length() > 0) { + if (!input.first().isEmpty()) { emitter.emit(Pair.of(input.first().substring(0, 1), input.second())); } } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/it/java/org/apache/crunch/lib/AvroTypeSortIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/lib/AvroTypeSortIT.java b/crunch-core/src/it/java/org/apache/crunch/lib/AvroTypeSortIT.java index a832a5d..918328b 100644 --- a/crunch-core/src/it/java/org/apache/crunch/lib/AvroTypeSortIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/lib/AvroTypeSortIT.java @@ -17,7 +17,7 @@ */ package org.apache.crunch.lib; -import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertEquals; import static org.apache.crunch.types.avro.Avros.ints; import static org.apache.crunch.types.avro.Avros.records; import static org.apache.crunch.types.avro.Avros.strings; @@ -118,7 +118,7 @@ public class AvroTypeSortIT implements Serializable { pipeline.done(); } - private void writeAvroFile(List people, File avroFile) throws IOException { + private static void writeAvroFile(List people, File avroFile) throws IOException { FileOutputStream outputStream = new FileOutputStream(avroFile); SpecificDatumWriter writer = new SpecificDatumWriter(Person.class); @@ -132,13 +132,12 @@ public class AvroTypeSortIT implements Serializable { outputStream.close(); } - private Person createPerson(String name, int age) throws IOException { + private static Person createPerson(String name, int age) { Person person = new Person(); person.age = age; person.name = name; - List siblingNames = Lists.newArrayList(); - person.siblingnames = siblingNames; + person.siblingnames = Lists.newArrayList(); return person; } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/it/java/org/apache/crunch/lib/SpecificAvroGroupByIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/lib/SpecificAvroGroupByIT.java b/crunch-core/src/it/java/org/apache/crunch/lib/SpecificAvroGroupByIT.java index 5292353..75ace34 100644 --- a/crunch-core/src/it/java/org/apache/crunch/lib/SpecificAvroGroupByIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/lib/SpecificAvroGroupByIT.java @@ -17,7 +17,7 @@ */ package org.apache.crunch.lib; -import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertEquals; import java.io.File; import java.io.FileOutputStream; @@ -97,14 +97,14 @@ public class SpecificAvroGroupByIT implements Serializable { pipeline.done(); } - private void createPersonAvroFile(File avroFile) throws IOException { + private static void createPersonAvroFile(File avroFile) throws IOException { Person person = new Person(); person.age = 40; person.name = "Bob"; List siblingNames = Lists.newArrayList(); - siblingNames.add("Bob" + "1"); - siblingNames.add("Bob" + "2"); + siblingNames.add("Bob1"); + siblingNames.add("Bob2"); person.siblingnames = siblingNames; FileOutputStream outputStream = new FileOutputStream(avroFile); http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/it/java/org/apache/crunch/lib/join/MapsideJoinStrategyIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/lib/join/MapsideJoinStrategyIT.java b/crunch-core/src/it/java/org/apache/crunch/lib/join/MapsideJoinStrategyIT.java index 09723d2..b5ce331 100644 --- a/crunch-core/src/it/java/org/apache/crunch/lib/join/MapsideJoinStrategyIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/lib/join/MapsideJoinStrategyIT.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Locale; import org.apache.crunch.MapFn; import org.apache.crunch.PTable; @@ -74,7 +75,7 @@ public class MapsideJoinStrategyIT { private static class CapOrdersFn extends MapFn { @Override public String map(String v) { - return v.toUpperCase(); + return v.toUpperCase(Locale.ENGLISH); } } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/PipelineResult.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/PipelineResult.java b/crunch-core/src/main/java/org/apache/crunch/PipelineResult.java index f98f305..c1ecdd3 100644 --- a/crunch-core/src/main/java/org/apache/crunch/PipelineResult.java +++ b/crunch-core/src/main/java/org/apache/crunch/PipelineResult.java @@ -143,7 +143,7 @@ public class PipelineResult { public boolean succeeded() { // return !stageResults.isEmpty(); - return this.status.equals(PipelineExecution.Status.SUCCEEDED); + return this.status == PipelineExecution.Status.SUCCEEDED; } public List getStageResults() { http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/TupleN.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/TupleN.java b/crunch-core/src/main/java/org/apache/crunch/TupleN.java index e5eceb5..aaf988a 100644 --- a/crunch-core/src/main/java/org/apache/crunch/TupleN.java +++ b/crunch-core/src/main/java/org/apache/crunch/TupleN.java @@ -26,7 +26,7 @@ import org.apache.commons.lang.builder.HashCodeBuilder; */ public class TupleN implements Tuple { - private final Object values[]; + private final Object[] values; public static TupleN of(Object... values) { return new TupleN(values); @@ -37,10 +37,12 @@ public class TupleN implements Tuple { System.arraycopy(values, 0, this.values, 0, values.length); } + @Override public Object get(int index) { return values[index]; } + @Override public int size() { return values.length; } @@ -56,12 +58,15 @@ public class TupleN implements Tuple { @Override public boolean equals(Object obj) { - if (this == obj) + if (this == obj) { return true; - if (obj == null) + } + if (obj == null) { return false; - if (getClass() != obj.getClass()) + } + if (getClass() != obj.getClass()) { return false; + } TupleN other = (TupleN) obj; return Arrays.equals(this.values, other.values); } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemTable.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemTable.java b/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemTable.java index 99405e6..a212ed1 100644 --- a/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemTable.java +++ b/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemTable.java @@ -189,7 +189,6 @@ public class MemTable extends MemCollection> implements PTable< return new MaterializableMap(this.materialize()); } - /** {@inheritDoc} */ @Override public PObject> asMap() { return new MapPObject(this); http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/impl/mr/collect/PGroupedTableImpl.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/collect/PGroupedTableImpl.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/collect/PGroupedTableImpl.java index e0e24ed..7eb2b09 100644 --- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/collect/PGroupedTableImpl.java +++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/collect/PGroupedTableImpl.java @@ -126,7 +126,7 @@ public class PGroupedTableImpl extends PCollectionImpl } @Override - protected void acceptInternal(PCollectionImpl.Visitor visitor) { + protected void acceptInternal(Visitor visitor) { visitor.visitGroupedTable(this); } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/impl/mr/emit/IntermediateEmitter.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/emit/IntermediateEmitter.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/emit/IntermediateEmitter.java index 955aed8..43033bb 100644 --- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/emit/IntermediateEmitter.java +++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/emit/IntermediateEmitter.java @@ -35,7 +35,6 @@ import com.google.common.collect.ImmutableList; public class IntermediateEmitter implements Emitter { private final List children; - private final Configuration conf; private final PType outputPType; private final boolean needDetachedValues; @@ -43,12 +42,11 @@ public class IntermediateEmitter implements Emitter { boolean disableDeepCopy) { this.outputPType = outputPType; this.children = ImmutableList.copyOf(children); - this.conf = conf; - outputPType.initialize(conf); needDetachedValues = !disableDeepCopy && this.children.size() > 1; } + @Override public void emit(Object emitted) { for (RTNode child : children) { Object value = emitted; @@ -59,6 +57,7 @@ public class IntermediateEmitter implements Emitter { } } + @Override public void flush() { // No-op } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DoNode.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DoNode.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DoNode.java index 7c64ab4..da40010 100644 --- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DoNode.java +++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DoNode.java @@ -146,7 +146,7 @@ public class DoNode { @Override public boolean equals(Object other) { - if (other == null || !(other instanceof DoNode)) { + if (!(other instanceof DoNode)) { return false; } if (this == other) { http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/Edge.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/Edge.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/Edge.java index 1e59df0..6eb50eb 100644 --- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/Edge.java +++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/Edge.java @@ -32,15 +32,12 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -/** - * - */ class Edge { private final Vertex head; private final Vertex tail; private final Set paths; - public Edge(Vertex head, Vertex tail) { + Edge(Vertex head, Vertex tail) { this.head = head; this.tail = tail; this.paths = Sets.newHashSet(); @@ -106,7 +103,7 @@ class Edge { @Override public boolean equals(Object other) { - if (other == null || !(other instanceof Edge)) { + if (!(other instanceof Edge)) { return false; } Edge e = (Edge) other; http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/PlanningParameters.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/PlanningParameters.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/PlanningParameters.java index d0b74b7..cdfb46f 100644 --- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/PlanningParameters.java +++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/PlanningParameters.java @@ -21,7 +21,7 @@ package org.apache.crunch.impl.mr.plan; * Collection of Configuration keys and various constants used when planning MapReduce jobs for a * pipeline. */ -public class PlanningParameters { +public final class PlanningParameters { public static final String MULTI_OUTPUT_PREFIX = "out"; http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/RuntimeParameters.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/RuntimeParameters.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/RuntimeParameters.java index a8e8aff..987ccd3 100644 --- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/RuntimeParameters.java +++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/RuntimeParameters.java @@ -20,7 +20,7 @@ package org.apache.crunch.impl.mr.run; /** * Parameters used during the runtime execution. */ -public class RuntimeParameters { +public final class RuntimeParameters { public static final String DEBUG = "crunch.debug"; http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniKeySource.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniKeySource.java b/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniKeySource.java index bee7ec1..4cc902a 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniKeySource.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniKeySource.java @@ -33,11 +33,10 @@ import java.io.IOException; public class TrevniKeySource extends FileSourceImpl implements ReadableSource { private static FormatBundle getBundle(AvroType ptype) { - FormatBundle bundle = FormatBundle.forInput(AvroTrevniKeyInputFormat.class) + return FormatBundle.forInput(AvroTrevniKeyInputFormat.class) .set(AvroJob.INPUT_IS_REFLECT, String.valueOf(ptype.hasReflect())) .set(AvroJob.INPUT_SCHEMA, ptype.getSchema().toString()) .set(Avros.REFLECT_DATA_FACTORY_CLASS, Avros.REFLECT_DATA_FACTORY.getClass().getName()); - return bundle; } public TrevniKeySource(Path path, AvroType ptype) { http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniKeyTarget.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniKeyTarget.java b/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniKeyTarget.java index e7acc08..5db11f0 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniKeyTarget.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniKeyTarget.java @@ -84,7 +84,7 @@ public class TrevniKeyTarget extends FileTargetImpl { Avros.configureReflectDataFactory(conf); configureForMapReduce(job, AvroKey.class, NullWritable.class, FormatBundle.forOutput(TrevniOutputFormat.class), - outputPath, name); + outputPath, null); } else { FormatBundle bundle = FormatBundle.forOutput( TrevniOutputFormat.class); http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniOutputFormat.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniOutputFormat.java b/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniOutputFormat.java index 9191788..a2b7dc9 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniOutputFormat.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/avro/trevni/TrevniOutputFormat.java @@ -20,18 +20,13 @@ package org.apache.crunch.io.avro.trevni; import java.io.IOException; import org.apache.avro.mapred.AvroKey; -import org.apache.avro.mapred.AvroValue; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -/** - * - */ public class TrevniOutputFormat extends FileOutputFormat, NullWritable> { - /** {@inheritDoc} */ @Override public RecordWriter, NullWritable> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/io/impl/AutoClosingIterator.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/impl/AutoClosingIterator.java b/crunch-core/src/main/java/org/apache/crunch/io/impl/AutoClosingIterator.java index 3bd802e..a854ebc 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/impl/AutoClosingIterator.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/impl/AutoClosingIterator.java @@ -39,11 +39,11 @@ public class AutoClosingIterator extends UnmodifiableIterator implements C @Override public boolean hasNext() { - if (!iter.hasNext()) { + if (iter.hasNext()) { + return true; + } else { Closeables.closeQuietly(this); return false; - } else { - return true; } } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/lib/Cogroup.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/lib/Cogroup.java b/crunch-core/src/main/java/org/apache/crunch/lib/Cogroup.java index 7f5f70d..514f46f 100644 --- a/crunch-core/src/main/java/org/apache/crunch/lib/Cogroup.java +++ b/crunch-core/src/main/java/org/apache/crunch/lib/Cogroup.java @@ -224,7 +224,7 @@ public class Cogroup { } PTable union = firstInter.union(inter); - PGroupedTable grouped = null; + PGroupedTable grouped; if (numReducers > 0) { grouped = union.groupByKey(numReducers); } else { @@ -240,7 +240,7 @@ public class Cogroup { private final int index; private final int size; - public CogroupFn(int index, int size) { + CogroupFn(int index, int size) { this.index = index; this.size = size; } @@ -259,7 +259,7 @@ public class Cogroup { private final TupleFactory factory; private final PType[] ptypes; - public PostGroupFn(TupleFactory tf, PType... ptypes) { + PostGroupFn(TupleFactory tf, PType... ptypes) { this.factory = tf; this.ptypes = ptypes; } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/lib/Distinct.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/lib/Distinct.java b/crunch-core/src/main/java/org/apache/crunch/lib/Distinct.java index 994830d..8100e09 100644 --- a/crunch-core/src/main/java/org/apache/crunch/lib/Distinct.java +++ b/crunch-core/src/main/java/org/apache/crunch/lib/Distinct.java @@ -86,7 +86,7 @@ public final class Distinct { private final int flushEvery; private final PType ptype; - public PreDistinctFn(int flushEvery, PType ptype) { + PreDistinctFn(int flushEvery, PType ptype) { this.flushEvery = flushEvery; this.ptype = ptype; } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/lib/Sample.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/lib/Sample.java b/crunch-core/src/main/java/org/apache/crunch/lib/Sample.java index 5a66101..5266545 100644 --- a/crunch-core/src/main/java/org/apache/crunch/lib/Sample.java +++ b/crunch-core/src/main/java/org/apache/crunch/lib/Sample.java @@ -120,6 +120,7 @@ public class Sample { PType> ptype = ptf.pairs(input.getPType(), ptf.ints()); return weightedReservoirSample( input.parallelDo(new MapFn>() { + @Override public Pair map(T t) { return Pair.of(t, 1); } }, ptype), sampleSize, @@ -160,7 +161,7 @@ public class Sample { return Pair.of(0, p); } }, ptf.tableOf(ptf.ints(), input.getPType())); - int[] ss = new int[] { sampleSize }; + int[] ss = { sampleSize }; return groupedWeightedReservoirSample(groupedIn, ss, seed) .parallelDo(new MapFn, T>() { @Override http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/lib/SampleUtils.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/lib/SampleUtils.java b/crunch-core/src/main/java/org/apache/crunch/lib/SampleUtils.java index 8769eed..701121b 100644 --- a/crunch-core/src/main/java/org/apache/crunch/lib/SampleUtils.java +++ b/crunch-core/src/main/java/org/apache/crunch/lib/SampleUtils.java @@ -33,7 +33,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -class SampleUtils { +final class SampleUtils { static class SampleFn extends FilterFn { @@ -41,9 +41,13 @@ class SampleUtils { private final double acceptanceProbability; private transient Random r; - public SampleFn(double acceptanceProbability, Long seed) { + SampleFn(double acceptanceProbability, Long seed) { Preconditions.checkArgument(0.0 < acceptanceProbability && acceptanceProbability < 1.0); - this.seed = seed == null ? System.currentTimeMillis() : seed; + if (seed == null) { + this.seed = System.currentTimeMillis(); + } else { + this.seed = seed; + } this.acceptanceProbability = acceptanceProbability; } @@ -64,13 +68,13 @@ class SampleUtils { static class ReservoirSampleFn extends DoFn>, Pair>> { - private int[] sampleSizes; - private Long seed; - private PType valueType; + private final int[] sampleSizes; + private final Long seed; + private final PType valueType; private transient List> reservoirs; private transient Random random; - public ReservoirSampleFn(int[] sampleSizes, Long seed, PType valueType) { + ReservoirSampleFn(int[] sampleSizes, Long seed, PType valueType) { this.sampleSizes = sampleSizes; this.seed = seed; this.valueType = valueType; @@ -80,7 +84,7 @@ class SampleUtils { public void initialize() { this.reservoirs = Lists.newArrayList(); this.valueType.initialize(getConfiguration()); - for (int i = 0; i < sampleSizes.length; i++) { + for (int sampleSize : sampleSizes) { reservoirs.add(Maps.newTreeMap()); } if (random == null) { @@ -113,7 +117,7 @@ class SampleUtils { @Override public void cleanup(Emitter>> emitter) { for (int id = 0; id < reservoirs.size(); id++) { - SortedMap reservoir = reservoirs.get(id); + Map reservoir = reservoirs.get(id); for (Map.Entry e : reservoir.entrySet()) { emitter.emit(Pair.of(id, Pair.of(e.getKey(), e.getValue()))); } @@ -123,11 +127,11 @@ class SampleUtils { static class WRSCombineFn extends CombineFn> { - private int[] sampleSizes; - private PType valueType; + private final int[] sampleSizes; + private final PType valueType; private List> reservoirs; - public WRSCombineFn(int[] sampleSizes, PType valueType) { + WRSCombineFn(int[] sampleSizes, PType valueType) { this.sampleSizes = sampleSizes; this.valueType = valueType; } @@ -135,7 +139,7 @@ class SampleUtils { @Override public void initialize() { this.reservoirs = Lists.newArrayList(); - for (int i = 0; i < sampleSizes.length; i++) { + for (int sampleSize : sampleSizes) { reservoirs.add(Maps.newTreeMap()); } this.valueType.initialize(getConfiguration()); @@ -158,7 +162,7 @@ class SampleUtils { @Override public void cleanup(Emitter>> emitter) { for (int i = 0; i < reservoirs.size(); i++) { - SortedMap reservoir = reservoirs.get(i); + Map reservoir = reservoirs.get(i); for (Map.Entry e : reservoir.entrySet()) { emitter.emit(Pair.of(i, Pair.of(e.getKey(), e.getValue()))); } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/lib/Sort.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/lib/Sort.java b/crunch-core/src/main/java/org/apache/crunch/lib/Sort.java index 1228617..94ce7d8 100644 --- a/crunch-core/src/main/java/org/apache/crunch/lib/Sort.java +++ b/crunch-core/src/main/java/org/apache/crunch/lib/Sort.java @@ -17,13 +17,10 @@ */ package org.apache.crunch.lib; -import static org.apache.crunch.lib.sort.SortFns.*; - import org.apache.avro.Schema; import org.apache.crunch.DoFn; import org.apache.crunch.Emitter; import org.apache.crunch.GroupingOptions; -import org.apache.crunch.GroupingOptions.Builder; import org.apache.crunch.PCollection; import org.apache.crunch.PTable; import org.apache.crunch.Pair; @@ -32,6 +29,7 @@ import org.apache.crunch.Tuple; import org.apache.crunch.Tuple3; import org.apache.crunch.Tuple4; import org.apache.crunch.TupleN; +import org.apache.crunch.lib.sort.SortFns; import org.apache.crunch.lib.sort.TotalOrderPartitioner; import org.apache.crunch.lib.sort.ReverseAvroComparator; import org.apache.crunch.lib.sort.ReverseWritableComparator; @@ -62,9 +60,9 @@ public class Sort { /** * To sort by column 2 ascending then column 1 descending, you would use: - * + * {@code * sortPairs(coll, by(2, ASCENDING), by(1, DESCENDING)) - * Column numbering is 1-based. + * } Column numbering is 1-based. */ public static class ColumnOrder { private int column; @@ -214,7 +212,7 @@ public class Sort { public static PCollection sortTuples(PCollection collection, int numReducers, ColumnOrder... columnOrders) { PType pType = collection.getPType(); - KeyExtraction ke = new KeyExtraction(pType, columnOrders); + SortFns.KeyExtraction ke = new SortFns.KeyExtraction(pType, columnOrders); PTable pt = collection.by(ke.getByFn(), ke.getKeyType()); Configuration conf = collection.getPipeline().getConfiguration(); GroupingOptions options = buildGroupingOptions(pt, conf, numReducers, columnOrders); @@ -226,7 +224,7 @@ public class Sort { int numReducers, Order order) { PType ptype = ptable.getKeyType(); PTypeFamily tf = ptable.getTypeFamily(); - Builder builder = GroupingOptions.builder(); + GroupingOptions.Builder builder = GroupingOptions.builder(); if (order == Order.DESCENDING) { if (tf == WritableTypeFamily.getInstance()) { builder.sortComparatorClass(ReverseWritableComparator.class); @@ -249,7 +247,7 @@ public class Sort { int numReducers, ColumnOrder[] columnOrders) { PTypeFamily tf = ptable.getTypeFamily(); PType keyType = ptable.getKeyType(); - Builder builder = GroupingOptions.builder(); + GroupingOptions.Builder builder = GroupingOptions.builder(); if (tf == WritableTypeFamily.getInstance()) { if (columnOrders.length == 1 && columnOrders[0].order == Order.DESCENDING) { builder.sortComparatorClass(ReverseWritableComparator.class); http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/lib/join/BloomFilterJoinStrategy.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/lib/join/BloomFilterJoinStrategy.java b/crunch-core/src/main/java/org/apache/crunch/lib/join/BloomFilterJoinStrategy.java index 4162169..6faef56 100644 --- a/crunch-core/src/main/java/org/apache/crunch/lib/join/BloomFilterJoinStrategy.java +++ b/crunch-core/src/main/java/org/apache/crunch/lib/join/BloomFilterJoinStrategy.java @@ -132,7 +132,7 @@ public class BloomFilterJoinStrategy implements JoinStrategy { throw new IllegalStateException("JoinType " + joinType + " is not supported for BloomFilter joins"); } - PTable filteredRightSide = null; + PTable filteredRightSide; if (left.getPipeline() instanceof MRPipeline) { PType bloomFilterType = getBloomFilterType(left.getTypeFamily()); PCollection bloomFilters = left.keys().parallelDo( @@ -175,7 +175,7 @@ public class BloomFilterJoinStrategy implements JoinStrategy { private transient MapFn keyToBytesFn; private PType ptype; - public CreateBloomFilterFn(int vectorSize, int nbHash, PType ptype) { + CreateBloomFilterFn(int vectorSize, int nbHash, PType ptype) { this.vectorSize = vectorSize; this.nbHash = nbHash; this.ptype = ptype; @@ -214,7 +214,7 @@ public class BloomFilterJoinStrategy implements JoinStrategy { private BloomFilter bloomFilter; private transient MapFn keyToBytesFn; - public FilterKeysWithBloomFilterFn(String inputPath, int vectorSize, int nbHash, PType keyType, PType bloomFilterPtype) { + FilterKeysWithBloomFilterFn(String inputPath, int vectorSize, int nbHash, PType keyType, PType bloomFilterPtype) { this.inputPath = inputPath; this.vectorSize = vectorSize; this.nbHash = nbHash; @@ -247,7 +247,7 @@ public class BloomFilterJoinStrategy implements JoinStrategy { ReadableSourceTarget sourceTarget = bloomFilterPType.getDefaultFileSource( getCacheFilePath()); - Iterable iterable = null; + Iterable iterable; try { iterable = sourceTarget.read(getConfiguration()); } catch (IOException e) { @@ -273,11 +273,11 @@ public class BloomFilterJoinStrategy implements JoinStrategy { private static MapFn getKeyToBytesMapFn(PType ptype, Configuration conf) { if (ptype instanceof AvroType) { return new AvroToBytesFn((AvroType)ptype, conf); - } else if (ptype instanceof WritableType) { + } + if (ptype instanceof WritableType) { return new WritableToBytesFn((WritableType)ptype, conf); - } else { - throw new IllegalStateException("Unrecognized PType: " + ptype); } + throw new IllegalStateException("Unrecognized PType: " + ptype); } /** @@ -302,7 +302,7 @@ public class BloomFilterJoinStrategy implements JoinStrategy { private WritableType ptype; private DataOutputBuffer dataOutputBuffer; - public WritableToBytesFn(WritableType ptype, Configuration conf) { + WritableToBytesFn(WritableType ptype, Configuration conf) { this.ptype = ptype; dataOutputBuffer = new DataOutputBuffer(); } @@ -332,7 +332,7 @@ public class BloomFilterJoinStrategy implements JoinStrategy { private BinaryEncoder encoder; private ReflectDatumWriter datumWriter; - public AvroToBytesFn(AvroType ptype, Configuration conf) { + AvroToBytesFn(AvroType ptype, Configuration conf) { this.ptype = ptype; datumWriter = Avros.getReflectDataFactory(conf).getWriter(ptype.getSchema()); } @@ -351,8 +351,6 @@ public class BloomFilterJoinStrategy implements JoinStrategy { return byteArrayOutputStream.toByteArray(); } - - } } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/lib/join/InnerJoinFn.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/lib/join/InnerJoinFn.java b/crunch-core/src/main/java/org/apache/crunch/lib/join/InnerJoinFn.java index a3d30d2..d1b1eff 100644 --- a/crunch-core/src/main/java/org/apache/crunch/lib/join/InnerJoinFn.java +++ b/crunch-core/src/main/java/org/apache/crunch/lib/join/InnerJoinFn.java @@ -41,7 +41,6 @@ public class InnerJoinFn extends JoinFn { super(keyType, leftValueType); } - /** {@inheritDoc} */ @Override public void initialize() { super.initialize(); @@ -49,7 +48,6 @@ public class InnerJoinFn extends JoinFn { this.leftValues = Lists.newArrayList(); } - /** {@inheritDoc} */ @Override public void join(K key, int id, Iterable> pairs, Emitter>> emitter) { if (!key.equals(lastKey)) { @@ -58,8 +56,9 @@ public class InnerJoinFn extends JoinFn { } if (id == 0) { // from left for (Pair pair : pairs) { - if (pair.first() != null) + if (pair.first() != null) { leftValues.add(leftValueType.getDetachedValue(pair.first())); + } } } else { // from right for (Pair pair : pairs) { @@ -70,7 +69,6 @@ public class InnerJoinFn extends JoinFn { } } - /** {@inheritDoc} */ @Override public String getJoinType() { return "innerJoin"; http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/lib/join/JoinUtils.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/lib/join/JoinUtils.java b/crunch-core/src/main/java/org/apache/crunch/lib/join/JoinUtils.java index 6efeccb..3f031cb 100644 --- a/crunch-core/src/main/java/org/apache/crunch/lib/join/JoinUtils.java +++ b/crunch-core/src/main/java/org/apache/crunch/lib/join/JoinUtils.java @@ -33,7 +33,6 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Partitioner; /** @@ -73,7 +72,7 @@ public class JoinUtils { @Override public int compare(TupleWritable o1, TupleWritable o2) { - return ((WritableComparable) o1.get(0)).compareTo((WritableComparable) o2.get(0)); + return ((Comparable) o1.get(0)).compareTo(o2.get(0)); } @Override http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/lib/sort/ReverseAvroComparator.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/lib/sort/ReverseAvroComparator.java b/crunch-core/src/main/java/org/apache/crunch/lib/sort/ReverseAvroComparator.java index ca6ba33..c404492 100644 --- a/crunch-core/src/main/java/org/apache/crunch/lib/sort/ReverseAvroComparator.java +++ b/crunch-core/src/main/java/org/apache/crunch/lib/sort/ReverseAvroComparator.java @@ -39,11 +39,11 @@ public class ReverseAvroComparator extends Configured implements RawComparato @Override public int compare(AvroKey o1, AvroKey o2) { - return -ReflectData.get().compare(o1.datum(), o2.datum(), schema); + return ReflectData.get().compare(o2.datum(), o1.datum(), schema); } @Override public int compare(byte[] arg0, int arg1, int arg2, byte[] arg3, int arg4, int arg5) { - return -BinaryData.compare(arg0, arg1, arg2, arg3, arg4, arg5, schema); + return BinaryData.compare(arg3, arg4, arg5, arg0, arg1, arg2, schema); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/lib/sort/SortFns.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/lib/sort/SortFns.java b/crunch-core/src/main/java/org/apache/crunch/lib/sort/SortFns.java index be218f6..86d3a20 100644 --- a/crunch-core/src/main/java/org/apache/crunch/lib/sort/SortFns.java +++ b/crunch-core/src/main/java/org/apache/crunch/lib/sort/SortFns.java @@ -165,7 +165,7 @@ public class SortFns { byFn = new SingleKeyFn(cols[0]); keyPType = pt.get(cols[0]); } else { - TupleFactory tf = null; + TupleFactory tf; switch (cols.length) { case 2: tf = TupleFactory.PAIR; http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableIterable.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableIterable.java b/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableIterable.java index 2dcc64f..aeb1fba 100644 --- a/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableIterable.java +++ b/crunch-core/src/main/java/org/apache/crunch/materialize/MaterializableIterable.java @@ -55,7 +55,8 @@ public class MaterializableIterable implements Iterable { public Path getPath() { if (source instanceof FileSourceImpl) { return ((FileSourceImpl) source).getPath(); - } else if (source instanceof PathTarget) { + } + if (source instanceof PathTarget) { return ((PathTarget) source).getPath(); } return null; http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/types/PTypeUtils.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/types/PTypeUtils.java b/crunch-core/src/main/java/org/apache/crunch/types/PTypeUtils.java index e61b98b..c8ccbd8 100644 --- a/crunch-core/src/main/java/org/apache/crunch/types/PTypeUtils.java +++ b/crunch-core/src/main/java/org/apache/crunch/types/PTypeUtils.java @@ -48,7 +48,7 @@ public class PTypeUtils { } else if (Tuple4.class.equals(typeClass)) { return tf.quads(tf.as(subTypes.get(0)), tf.as(subTypes.get(1)), tf.as(subTypes.get(2)), tf.as(subTypes.get(3))); } else if (TupleN.class.equals(typeClass)) { - PType[] newPTypes = subTypes.toArray(new PType[0]); + PType[] newPTypes = subTypes.toArray(new PType[subTypes.size()]); for (int i = 0; i < newPTypes.length; i++) { newPTypes[i] = tf.as(subTypes.get(i)); } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java b/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java index 546719c..cbb9c7c 100644 --- a/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java +++ b/crunch-core/src/main/java/org/apache/crunch/types/PTypes.java @@ -62,17 +62,19 @@ public class PTypes { return typeFamily.derived(clazz, new ThriftInputMapFn(clazz), new ThriftOutputMapFn(), typeFamily.bytes()); } - public static final PType enums(final Class type, PTypeFamily typeFamily) { + public static PType enums(Class type, PTypeFamily typeFamily) { return typeFamily.derived(type, new EnumInputMapper(type), new EnumOutputMapper(), typeFamily.strings()); } - public static MapFn BYTE_TO_BIGINT = new MapFn() { + public static final MapFn BYTE_TO_BIGINT = new MapFn() { + @Override public BigInteger map(ByteBuffer input) { return input == null ? null : new BigInteger(input.array()); } }; - public static MapFn BIGINT_TO_BYTE = new MapFn() { + public static final MapFn BIGINT_TO_BYTE = new MapFn() { + @Override public ByteBuffer map(BigInteger input) { return input == null ? null : ByteBuffer.wrap(input.toByteArray()); } @@ -83,7 +85,7 @@ public class PTypes { private final Class clazz; private transient ObjectMapper mapper; - public JacksonInputMapFn(Class clazz) { + JacksonInputMapFn(Class clazz) { this.clazz = clazz; } @@ -126,7 +128,7 @@ public class PTypes { private final Class clazz; private transient T instance; - public ProtoInputMapFn(Class clazz) { + ProtoInputMapFn(Class clazz) { this.clazz = clazz; } @@ -147,7 +149,7 @@ public class PTypes { private static class ProtoOutputMapFn extends MapFn { - public ProtoOutputMapFn() { + ProtoOutputMapFn() { } @Override @@ -163,7 +165,7 @@ public class PTypes { private transient TDeserializer deserializer; private transient byte[] bytes; - public ThriftInputMapFn(Class clazz) { + ThriftInputMapFn(Class clazz) { this.clazz = clazz; } @@ -195,7 +197,7 @@ public class PTypes { private transient TSerializer serializer; - public ThriftOutputMapFn() { + ThriftOutputMapFn() { } @Override @@ -216,7 +218,7 @@ public class PTypes { private static class EnumInputMapper extends MapFn { private final Class type; - public EnumInputMapper(Class type) { + EnumInputMapper(Class type) { this.type = type; } @@ -224,7 +226,7 @@ public class PTypes { public T map(String input) { return (T) Enum.valueOf(type, input); } - }; + } private static class EnumOutputMapper extends MapFn { @@ -232,16 +234,16 @@ public class PTypes { public String map(T input) { return input.name(); } - }; - - private static MapFn BYTE_TO_UUID = new MapFn() { + } + + private static final MapFn BYTE_TO_UUID = new MapFn() { @Override public UUID map(ByteBuffer input) { return new UUID(input.getLong(), input.getLong()); } }; - private static MapFn UUID_TO_BYTE = new MapFn() { + private static final MapFn UUID_TO_BYTE = new MapFn() { @Override public ByteBuffer map(UUID input) { ByteBuffer bb = ByteBuffer.wrap(new byte[16]); http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroTextOutputFormat.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroTextOutputFormat.java b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroTextOutputFormat.java index 4930235..1efb6fa 100644 --- a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroTextOutputFormat.java +++ b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroTextOutputFormat.java @@ -28,7 +28,7 @@ public class AvroTextOutputFormat extends TextOutputFormat { class DatumRecordTextWriter extends RecordWriter { private RecordWriter lineRecordWriter; - public DatumRecordTextWriter(RecordWriter recordWriter) { + DatumRecordTextWriter(RecordWriter recordWriter) { this.lineRecordWriter = recordWriter; } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableGroupedTableType.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableGroupedTableType.java b/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableGroupedTableType.java index 84318d3..8823541 100644 --- a/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableGroupedTableType.java +++ b/crunch-core/src/main/java/org/apache/crunch/types/writable/WritableGroupedTableType.java @@ -32,7 +32,7 @@ class WritableGroupedTableType extends PGroupedTableType { private final MapFn outputFn; private final Converter converter; - public WritableGroupedTableType(WritableTableType tableType) { + WritableGroupedTableType(WritableTableType tableType) { super(tableType); WritableType keyType = (WritableType) tableType.getKeyType(); WritableType valueType = (WritableType) tableType.getValueType(); http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobNameBuilderTest.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobNameBuilderTest.java b/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobNameBuilderTest.java index 3ba2763..eef318e 100644 --- a/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobNameBuilderTest.java +++ b/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobNameBuilderTest.java @@ -32,8 +32,8 @@ public class JobNameBuilderTest { @Test public void testBuild() { - final String pipelineName = "PipelineName"; - final String nodeName = "outputNode"; + String pipelineName = "PipelineName"; + String nodeName = "outputNode"; DoNode doNode = createDoNode(nodeName); JobNameBuilder jobNameBuilder = new JobNameBuilder(CONF, pipelineName, 1, 1); jobNameBuilder.visit(Lists.newArrayList(doNode)); @@ -44,8 +44,8 @@ public class JobNameBuilderTest { @Test public void testNodeNameTooLong() { - final String pipelineName = "PipelineName"; - final String nodeName = Strings.repeat("very_long_node_name", 100); + String pipelineName = "PipelineName"; + String nodeName = Strings.repeat("very_long_node_name", 100); DoNode doNode = createDoNode(nodeName); JobNameBuilder jobNameBuilder = new JobNameBuilder(CONF, pipelineName, 1, 1); jobNameBuilder.visit(Lists.newArrayList(doNode)); @@ -54,7 +54,7 @@ public class JobNameBuilderTest { assertFalse(jobName.contains(nodeName)); // Tests that the very long node name was shorten } - private DoNode createDoNode(String nodeName) { + private static DoNode createDoNode(String nodeName) { return DoNode.createOutputNode(nodeName, Writables.strings().getConverter(), Writables.strings()); } } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/test/java/org/apache/crunch/io/avro/AvroFileReaderFactoryTest.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/test/java/org/apache/crunch/io/avro/AvroFileReaderFactoryTest.java b/crunch-core/src/test/java/org/apache/crunch/io/avro/AvroFileReaderFactoryTest.java index 62085f8..0470477 100644 --- a/crunch-core/src/test/java/org/apache/crunch/io/avro/AvroFileReaderFactoryTest.java +++ b/crunch-core/src/test/java/org/apache/crunch/io/avro/AvroFileReaderFactoryTest.java @@ -29,7 +29,6 @@ import java.util.List; import org.apache.avro.Schema; import org.apache.avro.file.DataFileWriter; import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericData.Record; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.GenericRecord; @@ -82,7 +81,7 @@ public class AvroFileReaderFactoryTest { } - private AvroFileReaderFactory createFileReaderFactory(AvroType avroType) { + private static AvroFileReaderFactory createFileReaderFactory(AvroType avroType) { return new AvroFileReaderFactory(avroType); } @@ -148,7 +147,7 @@ public class AvroFileReaderFactoryTest { @Test public void testCreateDatumReader_Generic() { - DatumReader datumReader = AvroFileReaderFactory.createDatumReader(Avros.generics(Person.SCHEMA$)); + DatumReader datumReader = AvroFileReaderFactory.createDatumReader(Avros.generics(Person.SCHEMA$)); assertEquals(GenericDatumReader.class, datumReader.getClass()); } http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/test/java/org/apache/crunch/types/TupleFactoryTest.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/test/java/org/apache/crunch/types/TupleFactoryTest.java b/crunch-core/src/test/java/org/apache/crunch/types/TupleFactoryTest.java index 0726be2..b485026 100644 --- a/crunch-core/src/test/java/org/apache/crunch/types/TupleFactoryTest.java +++ b/crunch-core/src/test/java/org/apache/crunch/types/TupleFactoryTest.java @@ -48,6 +48,7 @@ public class TupleFactoryTest { assertEquals(TupleFactory.TUPLEN, TupleFactory.getTupleFactory(TupleN.class)); } + @Test public void testGetTupleFactory_CustomTupleClass() { TupleFactory customTupleFactory = TupleFactory.create(CustomTupleImplementation.class); assertEquals(customTupleFactory, TupleFactory.getTupleFactory(CustomTupleImplementation.class)); http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroSpecificDeepCopierClassloaderTest.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroSpecificDeepCopierClassloaderTest.java b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroSpecificDeepCopierClassloaderTest.java index aed7fce..11b0a78 100644 --- a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroSpecificDeepCopierClassloaderTest.java +++ b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroSpecificDeepCopierClassloaderTest.java @@ -35,7 +35,7 @@ public class AvroSpecificDeepCopierClassloaderTest { Person person = new Person(); person.name = "John Doe"; person.age = 42; - person.siblingnames = Lists. newArrayList(); + person.siblingnames = Lists.newArrayList(); Person deepCopyPerson = new AvroSpecificDeepCopier(Person.class, Person.SCHEMA$) .deepCopy(person); http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-examples/src/main/java/org/apache/crunch/examples/SortExample.java ---------------------------------------------------------------------- diff --git a/crunch-examples/src/main/java/org/apache/crunch/examples/SortExample.java b/crunch-examples/src/main/java/org/apache/crunch/examples/SortExample.java index a7cd8b7..9f230ab 100644 --- a/crunch-examples/src/main/java/org/apache/crunch/examples/SortExample.java +++ b/crunch-examples/src/main/java/org/apache/crunch/examples/SortExample.java @@ -28,10 +28,6 @@ import org.apache.hadoop.util.ToolRunner; * Simple Crunch tool for running sorting examples from the command line. */ public class SortExample extends CrunchTool { - - public SortExample() { - super(); - } @Override public int run(String[] args) throws Exception { http://git-wip-us.apache.org/repos/asf/crunch/blob/e0dbebf1/crunch-examples/src/main/java/org/apache/crunch/examples/WordAggregationHBase.java ---------------------------------------------------------------------- diff --git a/crunch-examples/src/main/java/org/apache/crunch/examples/WordAggregationHBase.java b/crunch-examples/src/main/java/org/apache/crunch/examples/WordAggregationHBase.java index 6214c4f..4c13078 100644 --- a/crunch-examples/src/main/java/org/apache/crunch/examples/WordAggregationHBase.java +++ b/crunch-examples/src/main/java/org/apache/crunch/examples/WordAggregationHBase.java @@ -19,10 +19,10 @@ package org.apache.crunch.examples; import java.io.IOException; import java.io.Serializable; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import com.google.common.collect.Lists; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.crunch.DoFn; @@ -62,18 +62,18 @@ import org.apache.hadoop.util.ToolRunner; */ @SuppressWarnings("serial") public class WordAggregationHBase extends Configured implements Tool, Serializable { - private final static Log LOG = LogFactory.getLog(WordAggregationHBase.class); + private static final Log LOG = LogFactory.getLog(WordAggregationHBase.class); // Configuration parameters. Here configured for a hbase instance running // locally - private static String HBASE_CONFIGURATION_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; - private static String HBASE_CONFIGURATION_ZOOKEEPER_CLIENTPORT = "hbase.zookeeper.property.clientPort"; - private static String hbaseZookeeperQuorum = "localhost"; - private static String hbaseZookeeperClientPort = "2181"; + private static final String HBASE_CONFIGURATION_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; + private static final String HBASE_CONFIGURATION_ZOOKEEPER_CLIENTPORT = "hbase.zookeeper.property.clientPort"; + private static final String hbaseZookeeperQuorum = "localhost"; + private static final String hbaseZookeeperClientPort = "2181"; // HBase parameters - private final String TABLE_SOURCE = "list"; - private final String TABLE_TARGET = "aggregation"; + private static final String TABLE_SOURCE = "list"; + private static final String TABLE_TARGET = "aggregation"; private final byte[] COLUMN_FAMILY_SOURCE = Bytes.toBytes("content"); private final byte[] COLUMN_QUALIFIER_SOURCE_PLAY = Bytes.toBytes("play"); @@ -83,7 +83,7 @@ public class WordAggregationHBase extends Configured implements Tool, Serializab private final byte[] COLUMN_QUALIFIER_TARGET_TEXT = Bytes.toBytes("text"); @Override - public int run(final String[] args) throws Exception { + public int run(String[] args) throws Exception { // We create the test rows first String type1 = "romeo and juliet"; String type2 = "macbeth"; @@ -145,7 +145,7 @@ public class WordAggregationHBase extends Configured implements Tool, Serializab * @param conf the hbase configuration * @throws IOException */ - private void putInHbase(final List putList, final Configuration conf) throws IOException { + private static void putInHbase(List putList, Configuration conf) throws IOException { HTable htable = new HTable(conf, TABLE_SOURCE); try { htable.put(putList); @@ -164,7 +164,7 @@ public class WordAggregationHBase extends Configured implements Tool, Serializab * @throws ZooKeeperConnectionException * @throws IOException */ - private void createTable(final Configuration conf, final String htableName, final String... families) throws MasterNotRunningException, ZooKeeperConnectionException, + private static void createTable(Configuration conf, String htableName, String... families) throws MasterNotRunningException, ZooKeeperConnectionException, IOException { HBaseAdmin hbase = new HBaseAdmin(conf); if (!hbase.tableExists(htableName)) { @@ -183,10 +183,9 @@ public class WordAggregationHBase extends Configured implements Tool, Serializab * @param character the rowkey * @param play the play (in column COLUMN_QUALIFIER_SOURCE_PLAY) * @param quote the quote (in column COLUMN_QUALIFIER_SOURCE_QUOTE) - * @return */ - private List createPuts(final List character, final List play, final List quote) throws IllegalArgumentException { - List list = new ArrayList(); + private List createPuts(List character, List play, List quote) { + List list = Lists.newArrayList(); if (character.size() != play.size() || quote.size() != play.size()) { LOG.error("Every list should have the same number of elements"); throw new IllegalArgumentException("Every list should have the same number of elements"); @@ -202,15 +201,15 @@ public class WordAggregationHBase extends Configured implements Tool, Serializab /** * Extract information from hbase - * + * * @param words the source from hbase - * @return a PTable composed of the type of the input as key + * @return a {@code PTable} composed of the type of the input as key * and its def as value */ - public PTable extractText(final PTable words) { + public PTable extractText(PTable words) { return words.parallelDo("Extract text", new DoFn, Pair>() { @Override - public void process(final Pair row, final Emitter> emitter) { + public void process(Pair row, Emitter> emitter) { byte[] type = row.second().getValue(COLUMN_FAMILY_SOURCE, COLUMN_QUALIFIER_SOURCE_PLAY); byte[] def = row.second().getValue(COLUMN_FAMILY_SOURCE, COLUMN_QUALIFIER_SOURCE_QUOTE); if (type != null && def != null) { @@ -229,19 +228,18 @@ public class WordAggregationHBase extends Configured implements Tool, Serializab * values for hbase. * @return a PCollection formed by the puts. */ - public PCollection createPut(final PTable extractedText) { + public PCollection createPut(PTable extractedText) { return extractedText.parallelDo("Convert to puts", new DoFn, Put>() { @Override - public void process(final Pair input, final Emitter emitter) { - Put put; - put = new Put(Bytes.toBytes(input.first())); + public void process(Pair input, Emitter emitter) { + Put put = new Put(Bytes.toBytes(input.first())); put.add(COLUMN_FAMILY_TARGET, COLUMN_QUALIFIER_TARGET_TEXT, Bytes.toBytes(input.second())); emitter.emit(put); } }, Writables.writables(Put.class)); } - public static void main(final String[] args) throws Exception { + public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); // Configuration hbase conf.set(HBASE_CONFIGURATION_ZOOKEEPER_QUORUM, hbaseZookeeperQuorum);