crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject git commit: CRUNCH-406: Have From/At code for Avro fields read PTypes and do the internal casting themselves
Date Sat, 31 May 2014 21:47:53 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 567d25dd7 -> c135bba63


CRUNCH-406: Have From/At code for Avro fields read PTypes and do the internal casting themselves


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/c135bba6
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/c135bba6
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/c135bba6

Branch: refs/heads/master
Commit: c135bba631c106f93aa2cb4f973bbff4262aba0d
Parents: 567d25d
Author: Josh Wills <jwills@apache.org>
Authored: Fri May 30 12:54:47 2014 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Sat May 31 14:43:10 2014 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/crunch/io/At.java  | 12 +++++-----
 .../main/java/org/apache/crunch/io/From.java    | 18 +++++++--------
 .../scala/org/apache/crunch/scrunch/IO.scala    | 24 ++++++++------------
 3 files changed, 24 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/c135bba6/crunch-core/src/main/java/org/apache/crunch/io/At.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/At.java b/crunch-core/src/main/java/org/apache/crunch/io/At.java
index e60831a..9a8db2f 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/At.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/At.java
@@ -131,22 +131,22 @@ public class At {
    * Creates a {@code SourceTarget<T>} instance from the Avro file(s) at the given
path name.
    * 
    * @param pathName The name of the path to the data on the filesystem
-   * @param avroType The {@code AvroType} for the Avro records
+   * @param ptype The {@code PType} for the Avro records
    * @return A new {@code SourceTarget<T>} instance
    */
-  public static <T> SourceTarget<T> avroFile(String pathName, AvroType<T>
avroType) {
-    return avroFile(new Path(pathName), avroType);
+  public static <T> SourceTarget<T> avroFile(String pathName, PType<T>
ptype) {
+    return avroFile(new Path(pathName), ptype);
   }
 
   /**
    * Creates a {@code SourceTarget<T>} instance from the Avro file(s) at the given
{@code Path}.
    * 
    * @param path The {@code Path} to the data
-   * @param avroType The {@code AvroType} for the Avro records
+   * @param ptype The {@code PType} for the Avro records
    * @return A new {@code SourceTarget<T>} instance
    */
-  public static <T> SourceTarget<T> avroFile(Path path, AvroType<T> avroType)
{
-    return new AvroFileSourceTarget<T>(path, avroType);
+  public static <T> SourceTarget<T> avroFile(Path path, PType<T> ptype)
{
+    return new AvroFileSourceTarget<T>(path, (AvroType<T>) ptype);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/crunch/blob/c135bba6/crunch-core/src/main/java/org/apache/crunch/io/From.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/From.java b/crunch-core/src/main/java/org/apache/crunch/io/From.java
index 14793a6..d72ddf4 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/From.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/From.java
@@ -221,33 +221,33 @@ public class From {
    * Creates a {@code Source<T>} instance from the Avro file(s) at the given path name.
    * 
    * @param pathName The name of the path to the data on the filesystem
-   * @param avroType The {@code AvroType} for the Avro records
+   * @param ptype The {@code AvroType} for the Avro records
    * @return A new {@code Source<T>} instance
    */
-  public static <T> Source<T> avroFile(String pathName, AvroType<T> avroType)
{
-    return avroFile(new Path(pathName), avroType);
+  public static <T> Source<T> avroFile(String pathName, PType<T> ptype)
{
+    return avroFile(new Path(pathName), ptype);
   }
 
   /**
    * Creates a {@code Source<T>} instance from the Avro file(s) at the given {@code
Path}.
    * 
    * @param path The {@code Path} to the data
-   * @param avroType The {@code AvroType} for the Avro records
+   * @param ptype The {@code AvroType} for the Avro records
    * @return A new {@code Source<T>} instance
    */
-  public static <T> Source<T> avroFile(Path path, AvroType<T> avroType)
{
-    return new AvroFileSource<T>(path, avroType);
+  public static <T> Source<T> avroFile(Path path, PType<T> ptype) {
+    return new AvroFileSource<T>(path, (AvroType<T>) ptype);
   }
 
   /**
    * Creates a {@code Source<T>} instance from the Avro file(s) at the given {@code
Path}s.
    *
    * @param paths A list of {@code Path}s to the data
-   * @param avroType The {@code AvroType} for the Avro records
+   * @param ptype The {@code PType} for the Avro records
    * @return A new {@code Source<T>} instance
    */
-  public static <T> Source<T> avroFile(List<Path> paths, AvroType<T>
avroType) {
-    return new AvroFileSource<T>(paths, avroType);
+  public static <T> Source<T> avroFile(List<Path> paths, PType<T>
ptype) {
+    return new AvroFileSource<T>(paths, (AvroType<T>) ptype);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/crunch/blob/c135bba6/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala
index c4af454..d9d1acd 100644
--- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala
@@ -20,19 +20,13 @@
 
 package org.apache.crunch.scrunch
 
-import org.apache.crunch.io.{From => from, To => to, At => at, SequentialFileNamingScheme}
+import org.apache.crunch.io.{From => from, To => to, At => at}
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.conf.Configuration
 import org.apache.crunch.types.PType
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
-import org.apache.crunch.types.avro.AvroType
 import org.apache.hadoop.io.Writable
 import org.apache.avro.specific.SpecificRecord
-import org.apache.crunch.Target
-import org.apache.crunch.io.impl.FileTargetImpl
-import org.apache.crunch.io.avro.AvroFileTarget
-import org.apache.crunch.io.seq.SeqFileTarget
-import org.apache.crunch.io.text.TextFileTarget
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
 
 object From {
@@ -80,16 +74,16 @@ object From {
     from.avroFile(paths, avroClass)
   }
 
-  def avroFile[T](pathName: String, avroType: AvroType[T]) = {
-    from.avroFile(pathName, avroType)
+  def avroFile[T](pathName: String, ptype: PType[T]) = {
+    from.avroFile(pathName, ptype)
   }
 
-  def avroFile[T](path: Path, avroType: AvroType[T]) = {
-    from.avroFile(path, avroType)
+  def avroFile[T](path: Path, ptype: PType[T]) = {
+    from.avroFile(path, ptype)
   }
 
-  def avroFile[T](paths: List[Path], avroType: AvroType[T]) = {
-    from.avroFile(paths, avroType)
+  def avroFile[T](paths: List[Path], ptype: PType[T]) = {
+    from.avroFile(paths, ptype)
   }
 
   def avroFile(pathName: String) = {
@@ -204,11 +198,11 @@ object At {
     at.avroFile(path, avroClass)
   }
 
-  def avroFile[T](pathName: String, avroType: AvroType[T]) = {
+  def avroFile[T](pathName: String, avroType: PType[T]) = {
     at.avroFile(pathName, avroType)
   }
 
-  def avroFile[T](path: Path, avroType: AvroType[T]) = {
+  def avroFile[T](path: Path, avroType: PType[T]) = {
     at.avroFile(path, avroType)
   }
 


Mime
View raw message