crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject git commit: CRUNCH-446: null checks inside of DoFns. Contributed by Allan Shoup.
Date Thu, 14 Aug 2014 16:56:17 GMT
Repository: crunch
Updated Branches:
  refs/heads/apache-crunch-0.8 54df53c1d -> a98c0e5d5


CRUNCH-446: null checks inside of DoFns. Contributed by Allan Shoup.


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/a98c0e5d
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/a98c0e5d
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/a98c0e5d

Branch: refs/heads/apache-crunch-0.8
Commit: a98c0e5d570bfa030f960920dfb510452081deff
Parents: 54df53c
Author: Josh Wills <jwills@apache.org>
Authored: Tue Aug 12 12:12:44 2014 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Thu Aug 14 08:42:26 2014 -0700

----------------------------------------------------------------------
 crunch-core/pom.xml                             |  6 +++
 .../src/main/java/org/apache/crunch/DoFn.java   | 48 ++++++++++++++++----
 .../org/apache/crunch/types/avro/Avros.java     |  4 +-
 pom.xml                                         |  6 +++
 4 files changed, 54 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/a98c0e5d/crunch-core/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-core/pom.xml b/crunch-core/pom.xml
index dd2403f..89b5c11 100644
--- a/crunch-core/pom.xml
+++ b/crunch-core/pom.xml
@@ -65,6 +65,12 @@ under the License.
     </dependency>
 
     <dependency>
+      <groupId>com.google.code.findbugs</groupId>
+      <artifactId>jsr305</artifactId>
+      <optional>true</optional>
+    </dependency>
+
+    <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client</artifactId>
       <scope>provided</scope>

http://git-wip-us.apache.org/repos/asf/crunch/blob/a98c0e5d/crunch-core/src/main/java/org/apache/crunch/DoFn.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/DoFn.java b/crunch-core/src/main/java/org/apache/crunch/DoFn.java
index c2ed35d..578f042 100644
--- a/crunch-core/src/main/java/org/apache/crunch/DoFn.java
+++ b/crunch-core/src/main/java/org/apache/crunch/DoFn.java
@@ -19,11 +19,16 @@ package org.apache.crunch;
 
 import java.io.Serializable;
 
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Counter;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.TaskInputOutputContext;
 
+import com.google.common.base.Preconditions;
+
 /**
  * Base class for all data processing functions in Crunch.
  * 
@@ -36,7 +41,11 @@ import org.apache.hadoop.mapreduce.TaskInputOutputContext;
  * 
  */
 public abstract class DoFn<S, T> implements Serializable {
+  /** This will be null prior to being set in {@link #setContext(TaskInputOutputContext)}.
*/
+  @CheckForNull
   private transient TaskInputOutputContext<?, ?, ?, ?> context;
+  /** This will be null prior to being set in {@link #setConfiguration(Configuration)}. */
+  @CheckForNull
   private transient Configuration conf;
 
   /**
@@ -100,9 +109,10 @@ public abstract class DoFn<S, T> implements Serializable {
 
   /**
    * Called during setup to pass the {@link TaskInputOutputContext} to this
-   * {@code DoFn} instance.
+   * {@code DoFn} instance. The specified {@code TaskInputOutputContext} must not be null.
    */
-  public void setContext(TaskInputOutputContext<?, ?, ?, ?> context) {
+  public void setContext(@Nonnull TaskInputOutputContext<?, ?, ?, ?> context) {
+    Preconditions.checkNotNull(context);
     this.context = context;
   }
 
@@ -110,9 +120,11 @@ public abstract class DoFn<S, T> implements Serializable {
    * Called during the setup of an initialized {@link org.apache.crunch.types.PType} that
    * relies on this instance.
    *
-   * @param conf The configuration for the {@code PType} being initialized
+   * @param conf
+   *          The non-null configuration for the {@code PType} being initialized
    */
-  public void setConfiguration(Configuration conf) {
+  public void setConfiguration(@Nonnull Configuration conf) {
+    Preconditions.checkNotNull(conf);
     this.conf = conf;
   }
 
@@ -164,6 +176,9 @@ public abstract class DoFn<S, T> implements Serializable {
    */
   @Deprecated
   protected Counter getCounter(Enum<?> counterName) {
+    if (context == null) {
+      return null;
+    }
     return context.getCounter(counterName);
   }
 
@@ -174,6 +189,9 @@ public abstract class DoFn<S, T> implements Serializable {
    */
   @Deprecated
   protected Counter getCounter(String groupName, String counterName) {
+    if (context == null) {
+        return null;
+    }
     return context.getCounter(groupName, counterName);
   }
 
@@ -182,7 +200,9 @@ public abstract class DoFn<S, T> implements Serializable {
   }
 
   protected void increment(String groupName, String counterName, long value) {
-    context.getCounter(groupName, counterName).increment(value);
+    if (context != null) {
+      context.getCounter(groupName, counterName).increment(value);
+    }
   }
 
   protected void increment(Enum<?> counterName) {
@@ -190,22 +210,34 @@ public abstract class DoFn<S, T> implements Serializable {
   }
 
   protected void increment(Enum<?> counterName, long value) {
-    context.getCounter(counterName).increment(value);
+    if (context != null) {
+      context.getCounter(counterName).increment(value);
+    }
   }
 
   protected void progress() {
-    context.progress();
+    if (context != null) {
+      context.progress();
+    }
   }
 
   protected TaskAttemptID getTaskAttemptID() {
+    if (context == null) {
+      return null;
+    }
     return context.getTaskAttemptID();
   }
 
   protected void setStatus(String status) {
-    context.setStatus(status);
+    if (context != null) {
+      context.setStatus(status);
+    }
   }
 
   protected String getStatus() {
+    if (context == null) {
+      return null;
+    }
     return context.getStatus();
   }
 

http://git-wip-us.apache.org/repos/asf/crunch/blob/a98c0e5d/crunch-core/src/main/java/org/apache/crunch/types/avro/Avros.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/avro/Avros.java b/crunch-core/src/main/java/org/apache/crunch/types/avro/Avros.java
index d6065f9..3ba4dfb 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/avro/Avros.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/avro/Avros.java
@@ -580,7 +580,7 @@ public class Avros {
     @Override
     public void setContext(TaskInputOutputContext<?, ?, ?, ?> context) {
       for (MapFn fn : fns) {
-        fn.setContext(getContext());
+        fn.setContext(context);
       }
     }
     
@@ -746,7 +746,7 @@ public class Avros {
     @Override
     public void setContext(TaskInputOutputContext<?, ?, ?, ?> context) {
       for (MapFn fn : fns) {
-        fn.setContext(getContext());
+        fn.setContext(context);
       }
     }
 

http://git-wip-us.apache.org/repos/asf/crunch/blob/a98c0e5d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8683080..8250712 100644
--- a/pom.xml
+++ b/pom.xml
@@ -94,6 +94,7 @@ under the License.
     <scala.version>2.10.4</scala.version>
     <scalatest.version>1.9.1</scalatest.version>
     <spark.version>0.9.1</spark.version>
+    <jsr305.version>1.3.9</jsr305.version>
   </properties>
 
   <scm>
@@ -393,6 +394,11 @@ under the License.
         <scope>test</scope>
       </dependency>
 
+      <dependency>
+        <groupId>com.google.code.findbugs</groupId>
+        <artifactId>jsr305</artifactId>
+        <version>${jsr305.version}</version>
+     </dependency>
     </dependencies>
     
   </dependencyManagement>


Mime
View raw message