crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gr...@apache.org
Subject git commit: CRUNCH-428 Disallow empty list of paths for inputs
Date Wed, 25 Jun 2014 08:18:38 GMT
Repository: crunch
Updated Branches:
  refs/heads/apache-crunch-0.8 31c7b6d8a -> 5b82e2612


CRUNCH-428 Disallow empty list of paths for inputs


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/5b82e261
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/5b82e261
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/5b82e261

Branch: refs/heads/apache-crunch-0.8
Commit: 5b82e2612dc621bad83b08e0fd1c79c6f444351d
Parents: 31c7b6d
Author: Gabriel Reid <greid@apache.org>
Authored: Wed Jun 25 10:10:13 2014 +0200
Committer: Gabriel Reid <greid@apache.org>
Committed: Wed Jun 25 10:12:40 2014 +0200

----------------------------------------------------------------------
 .../main/java/org/apache/crunch/io/From.java    |  8 ++--
 .../apache/crunch/io/impl/FileSourceImpl.java   | 15 +++---
 .../apache/crunch/io/text/TextFileSource.java   |  7 ++-
 .../java/org/apache/crunch/io/FromTest.java     | 48 ++++++++++++++++++++
 4 files changed, 63 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/5b82e261/crunch-core/src/main/java/org/apache/crunch/io/From.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/From.java b/crunch-core/src/main/java/org/apache/crunch/io/From.java
index d72ddf4..3c892a6 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/From.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/From.java
@@ -17,6 +17,10 @@
  */
 package org.apache.crunch.io;
 
+import java.io.IOException;
+import java.util.List;
+
+import com.google.common.base.Preconditions;
 import org.apache.avro.Schema;
 import org.apache.avro.file.DataFileReader;
 import org.apache.avro.generic.GenericData;
@@ -45,9 +49,6 @@ import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 
-import java.io.IOException;
-import java.util.List;
-
 /**
  * <p>Static factory methods for creating common {@link Source} types.</p>
  * 
@@ -311,6 +312,7 @@ public class From {
    * @return A new {@code Source<GenericData.Record>} instance
    */
   public static Source<GenericData.Record> avroFile(List<Path> paths, Configuration
conf) {
+    Preconditions.checkArgument(!paths.isEmpty(), "At least one path must be supplied");
     return avroFile(paths, Avros.generics(getSchemaFromPath(paths.get(0), conf)));
   }
 

http://git-wip-us.apache.org/repos/asf/crunch/blob/5b82e261/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java b/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java
index 1151ad5..9917bad 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java
@@ -17,11 +17,12 @@
  */
 package org.apache.crunch.io.impl;
 
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
 import java.io.IOException;
-
 import java.util.List;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
 import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -39,7 +40,6 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 
 public class FileSourceImpl<T> implements Source<T> {
 
@@ -64,7 +64,8 @@ public class FileSourceImpl<T> implements Source<T> {
   }
 
   public FileSourceImpl(List<Path> paths, PType<T> ptype, FormatBundle<? extends
InputFormat> inputBundle) {
-    this.path = paths.isEmpty() ? null : paths.get(0);
+    Preconditions.checkArgument(!paths.isEmpty(), "Must supply at least one input path");
+    this.path = paths.get(0);
     this.paths = paths;
     this.ptype = ptype;
     this.inputBundle = inputBundle;
@@ -72,9 +73,7 @@ public class FileSourceImpl<T> implements Source<T> {
 
   @Deprecated
   public Path getPath() {
-    if (paths.isEmpty()) {
-      return null;
-    } else if (paths.size() > 1) {
+    if (paths.size() > 1) {
       LOG.warn("getPath() called for source with multiple paths, only " +
           "returning first. Source: " + this);
     }

http://git-wip-us.apache.org/repos/asf/crunch/blob/5b82e261/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java b/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java
index 732288d..214ab78 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java
@@ -18,13 +18,12 @@
 package org.apache.crunch.io.text;
 
 import java.io.IOException;
-
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.crunch.ReadableData;
 import org.apache.crunch.impl.mr.run.RuntimeParameters;
 import org.apache.crunch.io.ReadableSource;
-import org.apache.crunch.ReadableData;
 import org.apache.crunch.io.impl.FileSourceImpl;
 import org.apache.crunch.types.PType;
 import org.apache.crunch.types.avro.AvroTypeFamily;
@@ -36,7 +35,7 @@ import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 
 public class TextFileSource<T> extends FileSourceImpl<T> implements ReadableSource<T>
{
 
-  private static <S> Class<? extends FileInputFormat<?, ?>> getInputFormat(Path
path, PType<S> ptype) {
+  private static <S> Class<? extends FileInputFormat<?, ?>> getInputFormat(PType<S>
ptype) {
     if (ptype.getFamily().equals(AvroTypeFamily.getInstance())) {
       return AvroUtf8InputFormat.class;
     } else {
@@ -49,7 +48,7 @@ public class TextFileSource<T> extends FileSourceImpl<T> implements
ReadableSour
   }
 
   public TextFileSource(List<Path> paths, PType<T> ptype) {
-    super(paths, ptype, getInputFormat(paths.get(0), ptype));
+    super(paths, ptype, getInputFormat(ptype));
     inputBundle.set(RuntimeParameters.DISABLE_COMBINE_FILE, Boolean.FALSE.toString());
   }
 

http://git-wip-us.apache.org/repos/asf/crunch/blob/5b82e261/crunch-core/src/test/java/org/apache/crunch/io/FromTest.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/test/java/org/apache/crunch/io/FromTest.java b/crunch-core/src/test/java/org/apache/crunch/io/FromTest.java
new file mode 100644
index 0000000..06ef6cd
--- /dev/null
+++ b/crunch-core/src/test/java/org/apache/crunch/io/FromTest.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.io;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.junit.Test;
+
+public class FromTest {
+
+  @Test(expected=IllegalArgumentException.class)
+  public void testAvroFile_EmptyPathListNotAllowed() {
+    From.avroFile(ImmutableList.<Path>of());
+  }
+
+  @Test(expected=IllegalArgumentException.class)
+  public void testTextFile_EmptyPathListNotAllowed() {
+    From.textFile(ImmutableList.<Path>of());
+  }
+
+  @Test(expected=IllegalArgumentException.class)
+  public void testFormattedFile_EmptyPathListNotAllowed() {
+    From.formattedFile(ImmutableList.<Path>of(), TextInputFormat.class, LongWritable.class,
Text.class);
+  }
+
+  @Test(expected=IllegalArgumentException.class)
+  public void testSequenceFile_EmptyPathListNotAllowed() {
+    From.sequenceFile(ImmutableList.<Path>of(), LongWritable.class, Text.class);
+  }
+}
\ No newline at end of file


Mime
View raw message