spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject git commit: SPARK-3422. JavaAPISuite.getHadoopInputSplits isn't used anywhere.
Date Tue, 09 Sep 2014 17:23:31 GMT
Repository: spark
Updated Branches:
  refs/heads/master 1e03cf79f -> 88547a09f


SPARK-3422. JavaAPISuite.getHadoopInputSplits isn't used anywhere.

Author: Sandy Ryza <sandy@cloudera.com>

Closes #2324 from sryza/sandy-spark-3422 and squashes the following commits:

6446175 [Sandy Ryza] SPARK-3422. JavaAPISuite.getHadoopInputSplits isn't used anywhere.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/88547a09
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/88547a09
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/88547a09

Branch: refs/heads/master
Commit: 88547a09fcc25df132b401ecec4ebe1ef6778576
Parents: 1e03cf7
Author: Sandy Ryza <sandy@cloudera.com>
Authored: Tue Sep 9 10:23:28 2014 -0700
Committer: Reynold Xin <rxin@apache.org>
Committed: Tue Sep 9 10:23:28 2014 -0700

----------------------------------------------------------------------
 .../java/org/apache/spark/JavaAPISuite.java     | 25 --------------------
 1 file changed, 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/88547a09/core/src/test/java/org/apache/spark/JavaAPISuite.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/spark/JavaAPISuite.java b/core/src/test/java/org/apache/spark/JavaAPISuite.java
index be99dc5..b8574df 100644
--- a/core/src/test/java/org/apache/spark/JavaAPISuite.java
+++ b/core/src/test/java/org/apache/spark/JavaAPISuite.java
@@ -29,19 +29,14 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
 import com.google.common.base.Optional;
 import com.google.common.base.Charsets;
 import com.google.common.io.Files;
 import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.compress.DefaultCodec;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-import org.apache.hadoop.mapred.TextInputFormat;
 import org.apache.hadoop.mapreduce.Job;
 import org.junit.After;
 import org.junit.Assert;
@@ -49,7 +44,6 @@ import org.junit.Before;
 import org.junit.Test;
 
 import org.apache.spark.api.java.JavaDoubleRDD;
-import org.apache.spark.api.java.JavaHadoopRDD;
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.api.java.JavaSparkContext;
@@ -1313,23 +1307,4 @@ public class JavaAPISuite implements Serializable {
     SomeCustomClass[] collected = (SomeCustomClass[]) rdd.rdd().retag(SomeCustomClass.class).collect();
     Assert.assertEquals(data.size(), collected.length);
   }
-
-  public void getHadoopInputSplits() {
-    String outDir = new File(tempDir, "output").getAbsolutePath();
-    sc.parallelize(Arrays.asList(1, 2, 3, 4, 5), 2).saveAsTextFile(outDir);
-
-    JavaHadoopRDD<LongWritable, Text> hadoopRDD = (JavaHadoopRDD<LongWritable, Text>)
-        sc.hadoopFile(outDir, TextInputFormat.class, LongWritable.class, Text.class);
-    List<String> inputPaths = hadoopRDD.mapPartitionsWithInputSplit(
-        new Function2<InputSplit, Iterator<Tuple2<LongWritable, Text>>, Iterator<String>>()
{
-      @Override
-      public Iterator<String> call(InputSplit split, Iterator<Tuple2<LongWritable,
Text>> it)
-          throws Exception {
-        FileSplit fileSplit = (FileSplit) split;
-        return Lists.newArrayList(fileSplit.getPath().toUri().getPath()).iterator();
-      }
-    }, true).collect();
-    Assert.assertEquals(Sets.newHashSet(inputPaths),
-        Sets.newHashSet(outDir + "/part-00000", outDir + "/part-00001"));
-  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message