Return-Path: X-Original-To: apmail-crunch-commits-archive@www.apache.org Delivered-To: apmail-crunch-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 816E9116A0 for ; Wed, 25 Jun 2014 08:18:39 +0000 (UTC) Received: (qmail 83911 invoked by uid 500); 25 Jun 2014 08:18:39 -0000 Delivered-To: apmail-crunch-commits-archive@crunch.apache.org Received: (qmail 83873 invoked by uid 500); 25 Jun 2014 08:18:39 -0000 Mailing-List: contact commits-help@crunch.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@crunch.apache.org Delivered-To: mailing list commits@crunch.apache.org Received: (qmail 83863 invoked by uid 99); 25 Jun 2014 08:18:39 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 25 Jun 2014 08:18:39 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id E113E912F13; Wed, 25 Jun 2014 08:18:38 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: greid@apache.org To: commits@crunch.apache.org Message-Id: <97b70b6eface4c6d9d89e1ffb10a4fcd@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: CRUNCH-428 Disallow empty list of paths for inputs Date: Wed, 25 Jun 2014 08:18:38 +0000 (UTC) Repository: crunch Updated Branches: refs/heads/apache-crunch-0.8 31c7b6d8a -> 5b82e2612 CRUNCH-428 Disallow empty list of paths for inputs Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/5b82e261 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/5b82e261 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/5b82e261 Branch: refs/heads/apache-crunch-0.8 Commit: 5b82e2612dc621bad83b08e0fd1c79c6f444351d Parents: 31c7b6d Author: Gabriel Reid Authored: Wed Jun 25 10:10:13 2014 +0200 Committer: Gabriel Reid Committed: Wed Jun 25 10:12:40 2014 +0200 ---------------------------------------------------------------------- .../main/java/org/apache/crunch/io/From.java | 8 ++-- .../apache/crunch/io/impl/FileSourceImpl.java | 15 +++--- .../apache/crunch/io/text/TextFileSource.java | 7 ++- .../java/org/apache/crunch/io/FromTest.java | 48 ++++++++++++++++++++ 4 files changed, 63 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/5b82e261/crunch-core/src/main/java/org/apache/crunch/io/From.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/From.java b/crunch-core/src/main/java/org/apache/crunch/io/From.java index d72ddf4..3c892a6 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/From.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/From.java @@ -17,6 +17,10 @@ */ package org.apache.crunch.io; +import java.io.IOException; +import java.util.List; + +import com.google.common.base.Preconditions; import org.apache.avro.Schema; import org.apache.avro.file.DataFileReader; import org.apache.avro.generic.GenericData; @@ -45,9 +49,6 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import java.io.IOException; -import java.util.List; - /** *

Static factory methods for creating common {@link Source} types.

* @@ -311,6 +312,7 @@ public class From { * @return A new {@code Source} instance */ public static Source avroFile(List paths, Configuration conf) { + Preconditions.checkArgument(!paths.isEmpty(), "At least one path must be supplied"); return avroFile(paths, Avros.generics(getSchemaFromPath(paths.get(0), conf))); } http://git-wip-us.apache.org/repos/asf/crunch/blob/5b82e261/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java b/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java index 1151ad5..9917bad 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java @@ -17,11 +17,12 @@ */ package org.apache.crunch.io.impl; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; import java.io.IOException; - import java.util.List; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -39,7 +40,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; public class FileSourceImpl implements Source { @@ -64,7 +64,8 @@ public class FileSourceImpl implements Source { } public FileSourceImpl(List paths, PType ptype, FormatBundle inputBundle) { - this.path = paths.isEmpty() ? null : paths.get(0); + Preconditions.checkArgument(!paths.isEmpty(), "Must supply at least one input path"); + this.path = paths.get(0); this.paths = paths; this.ptype = ptype; this.inputBundle = inputBundle; @@ -72,9 +73,7 @@ public class FileSourceImpl implements Source { @Deprecated public Path getPath() { - if (paths.isEmpty()) { - return null; - } else if (paths.size() > 1) { + if (paths.size() > 1) { LOG.warn("getPath() called for source with multiple paths, only " + "returning first. Source: " + this); } http://git-wip-us.apache.org/repos/asf/crunch/blob/5b82e261/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java b/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java index 732288d..214ab78 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java @@ -18,13 +18,12 @@ package org.apache.crunch.io.text; import java.io.IOException; - import java.util.Collections; import java.util.List; +import org.apache.crunch.ReadableData; import org.apache.crunch.impl.mr.run.RuntimeParameters; import org.apache.crunch.io.ReadableSource; -import org.apache.crunch.ReadableData; import org.apache.crunch.io.impl.FileSourceImpl; import org.apache.crunch.types.PType; import org.apache.crunch.types.avro.AvroTypeFamily; @@ -36,7 +35,7 @@ import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; public class TextFileSource extends FileSourceImpl implements ReadableSource { - private static Class> getInputFormat(Path path, PType ptype) { + private static Class> getInputFormat(PType ptype) { if (ptype.getFamily().equals(AvroTypeFamily.getInstance())) { return AvroUtf8InputFormat.class; } else { @@ -49,7 +48,7 @@ public class TextFileSource extends FileSourceImpl implements ReadableSour } public TextFileSource(List paths, PType ptype) { - super(paths, ptype, getInputFormat(paths.get(0), ptype)); + super(paths, ptype, getInputFormat(ptype)); inputBundle.set(RuntimeParameters.DISABLE_COMBINE_FILE, Boolean.FALSE.toString()); } http://git-wip-us.apache.org/repos/asf/crunch/blob/5b82e261/crunch-core/src/test/java/org/apache/crunch/io/FromTest.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/test/java/org/apache/crunch/io/FromTest.java b/crunch-core/src/test/java/org/apache/crunch/io/FromTest.java new file mode 100644 index 0000000..06ef6cd --- /dev/null +++ b/crunch-core/src/test/java/org/apache/crunch/io/FromTest.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.io; + +import com.google.common.collect.ImmutableList; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.junit.Test; + +public class FromTest { + + @Test(expected=IllegalArgumentException.class) + public void testAvroFile_EmptyPathListNotAllowed() { + From.avroFile(ImmutableList.of()); + } + + @Test(expected=IllegalArgumentException.class) + public void testTextFile_EmptyPathListNotAllowed() { + From.textFile(ImmutableList.of()); + } + + @Test(expected=IllegalArgumentException.class) + public void testFormattedFile_EmptyPathListNotAllowed() { + From.formattedFile(ImmutableList.of(), TextInputFormat.class, LongWritable.class, Text.class); + } + + @Test(expected=IllegalArgumentException.class) + public void testSequenceFile_EmptyPathListNotAllowed() { + From.sequenceFile(ImmutableList.of(), LongWritable.class, Text.class); + } +} \ No newline at end of file