accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From afu...@apache.org
Subject svn commit: r1483417 - in /accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce: AccumuloFileOutputFormat.java AccumuloOutputFormat.java InputFormatBase.java
Date Thu, 16 May 2013 15:49:33 GMT
Author: afuchs
Date: Thu May 16 15:49:33 2013
New Revision: 1483417

URL: http://svn.apache.org/r1483417
Log:
ACCUMULO-1421 added reflection to improve runtime compatibility between hadoop 1 and 2 JobContext
objects

Modified:
    accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormat.java
    accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
    accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java

Modified: accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormat.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormat.java?rev=1483417&r1=1483416&r2=1483417&view=diff
==============================================================================
--- accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormat.java
(original)
+++ accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormat.java
Thu May 16 15:49:33 2013
@@ -65,7 +65,7 @@ public class AccumuloFileOutputFormat ex
    * @since 1.5.0
    */
   protected static AccumuloConfiguration getAccumuloConfiguration(JobContext context) {
-    return FileOutputConfigurator.getAccumuloConfiguration(CLASS, context.getConfiguration());
+    return FileOutputConfigurator.getAccumuloConfiguration(CLASS, InputFormatBase.getConfiguration(context));
   }
   
   /**
@@ -141,7 +141,7 @@ public class AccumuloFileOutputFormat ex
   @Override
   public RecordWriter<Key,Value> getRecordWriter(TaskAttemptContext context) throws
IOException {
     // get the path of the temporary output file
-    final Configuration conf = context.getConfiguration();
+    final Configuration conf = InputFormatBase.getConfiguration(context);
     final AccumuloConfiguration acuConf = getAccumuloConfiguration(context);
     
     final String extension = acuConf.get(Property.TABLE_FILE_TYPE);

Modified: accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java?rev=1483417&r1=1483416&r2=1483417&view=diff
==============================================================================
--- accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
(original)
+++ accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
Thu May 16 15:49:33 2013
@@ -104,7 +104,7 @@ public class AccumuloOutputFormat extend
    * @see #setConnectorInfo(Job, String, AuthenticationToken)
    */
   protected static Boolean isConnectorInfoSet(JobContext context) {
-    return OutputConfigurator.isConnectorInfoSet(CLASS, context.getConfiguration());
+    return OutputConfigurator.isConnectorInfoSet(CLASS, InputFormatBase.getConfiguration(context));
   }
   
   /**
@@ -117,7 +117,7 @@ public class AccumuloOutputFormat extend
    * @see #setConnectorInfo(Job, String, AuthenticationToken)
    */
   protected static String getPrincipal(JobContext context) {
-    return OutputConfigurator.getPrincipal(CLASS, context.getConfiguration());
+    return OutputConfigurator.getPrincipal(CLASS, InputFormatBase.getConfiguration(context));
   }
   
   /**
@@ -130,7 +130,7 @@ public class AccumuloOutputFormat extend
    * @see #setConnectorInfo(Job, String, AuthenticationToken)
    */
   protected static String getTokenClass(JobContext context) {
-    return OutputConfigurator.getTokenClass(CLASS, context.getConfiguration());
+    return OutputConfigurator.getTokenClass(CLASS, InputFormatBase.getConfiguration(context));
   }
   
   /**
@@ -144,7 +144,7 @@ public class AccumuloOutputFormat extend
    * @see #setConnectorInfo(Job, String, AuthenticationToken)
    */
   protected static byte[] getToken(JobContext context) {
-    return OutputConfigurator.getToken(CLASS, context.getConfiguration());
+    return OutputConfigurator.getToken(CLASS, InputFormatBase.getConfiguration(context));
   }
   
   /**
@@ -186,7 +186,7 @@ public class AccumuloOutputFormat extend
    * @see #setMockInstance(Job, String)
    */
   protected static Instance getInstance(JobContext context) {
-    return OutputConfigurator.getInstance(CLASS, context.getConfiguration());
+    return OutputConfigurator.getInstance(CLASS, InputFormatBase.getConfiguration(context));
   }
   
   /**
@@ -212,7 +212,7 @@ public class AccumuloOutputFormat extend
    * @see #setLogLevel(Job, Level)
    */
   protected static Level getLogLevel(JobContext context) {
-    return OutputConfigurator.getLogLevel(CLASS, context.getConfiguration());
+    return OutputConfigurator.getLogLevel(CLASS, InputFormatBase.getConfiguration(context));
   }
   
   /**
@@ -239,7 +239,7 @@ public class AccumuloOutputFormat extend
    * @see #setDefaultTableName(Job, String)
    */
   protected static String getDefaultTableName(JobContext context) {
-    return OutputConfigurator.getDefaultTableName(CLASS, context.getConfiguration());
+    return OutputConfigurator.getDefaultTableName(CLASS, InputFormatBase.getConfiguration(context));
   }
   
   /**
@@ -266,7 +266,7 @@ public class AccumuloOutputFormat extend
    * @see #setBatchWriterOptions(Job, BatchWriterConfig)
    */
   protected static BatchWriterConfig getBatchWriterOptions(JobContext context) {
-    return OutputConfigurator.getBatchWriterOptions(CLASS, context.getConfiguration());
+    return OutputConfigurator.getBatchWriterOptions(CLASS, InputFormatBase.getConfiguration(context));
   }
   
   /**
@@ -295,7 +295,7 @@ public class AccumuloOutputFormat extend
    * @see #setCreateTables(Job, boolean)
    */
   protected static Boolean canCreateTables(JobContext context) {
-    return OutputConfigurator.canCreateTables(CLASS, context.getConfiguration());
+    return OutputConfigurator.canCreateTables(CLASS, InputFormatBase.getConfiguration(context));
   }
   
   /**
@@ -324,7 +324,7 @@ public class AccumuloOutputFormat extend
    * @see #setSimulationMode(Job, boolean)
    */
   protected static Boolean getSimulationMode(JobContext context) {
-    return OutputConfigurator.getSimulationMode(CLASS, context.getConfiguration());
+    return OutputConfigurator.getSimulationMode(CLASS, InputFormatBase.getConfiguration(context));
   }
   
   /**

Modified: accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java?rev=1483417&r1=1483416&r2=1483417&view=diff
==============================================================================
--- accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
(original)
+++ accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
Thu May 16 15:49:33 2013
@@ -20,6 +20,7 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Method;
 import java.math.BigInteger;
 import java.net.InetAddress;
 import java.net.URLDecoder;
@@ -126,7 +127,7 @@ public abstract class InputFormatBase<K,
    * @see #setConnectorInfo(Job, String, AuthenticationToken)
    */
   protected static Boolean isConnectorInfoSet(JobContext context) {
-    return InputConfigurator.isConnectorInfoSet(CLASS, context.getConfiguration());
+    return InputConfigurator.isConnectorInfoSet(CLASS, getConfiguration(context));
   }
   
   /**
@@ -139,7 +140,7 @@ public abstract class InputFormatBase<K,
    * @see #setConnectorInfo(Job, String, AuthenticationToken)
    */
   protected static String getPrincipal(JobContext context) {
-    return InputConfigurator.getPrincipal(CLASS, context.getConfiguration());
+    return InputConfigurator.getPrincipal(CLASS, getConfiguration(context));
   }
   
   /**
@@ -152,7 +153,7 @@ public abstract class InputFormatBase<K,
    * @see #setConnectorInfo(Job, String, AuthenticationToken)
    */
   protected static String getTokenClass(JobContext context) {
-    return InputConfigurator.getTokenClass(CLASS, context.getConfiguration());
+    return InputConfigurator.getTokenClass(CLASS, getConfiguration(context));
   }
   
   /**
@@ -166,7 +167,7 @@ public abstract class InputFormatBase<K,
    * @see #setConnectorInfo(Job, String, AuthenticationToken)
    */
   protected static byte[] getToken(JobContext context) {
-    return InputConfigurator.getToken(CLASS, context.getConfiguration());
+    return InputConfigurator.getToken(CLASS, getConfiguration(context));
   }
   
   /**
@@ -208,7 +209,7 @@ public abstract class InputFormatBase<K,
    * @see #setMockInstance(Job, String)
    */
   protected static Instance getInstance(JobContext context) {
-    return InputConfigurator.getInstance(CLASS, context.getConfiguration());
+    return InputConfigurator.getInstance(CLASS, getConfiguration(context));
   }
   
   /**
@@ -234,7 +235,7 @@ public abstract class InputFormatBase<K,
    * @see #setLogLevel(Job, Level)
    */
   protected static Level getLogLevel(JobContext context) {
-    return InputConfigurator.getLogLevel(CLASS, context.getConfiguration());
+    return InputConfigurator.getLogLevel(CLASS, getConfiguration(context));
   }
   
   /**
@@ -260,7 +261,7 @@ public abstract class InputFormatBase<K,
    * @see #setInputTableName(Job, String)
    */
   protected static String getInputTableName(JobContext context) {
-    return InputConfigurator.getInputTableName(CLASS, context.getConfiguration());
+    return InputConfigurator.getInputTableName(CLASS, getConfiguration(context));
   }
   
   /**
@@ -286,7 +287,7 @@ public abstract class InputFormatBase<K,
    * @see #setScanAuthorizations(Job, Authorizations)
    */
   protected static Authorizations getScanAuthorizations(JobContext context) {
-    return InputConfigurator.getScanAuthorizations(CLASS, context.getConfiguration());
+    return InputConfigurator.getScanAuthorizations(CLASS, getConfiguration(context));
   }
   
   /**
@@ -314,7 +315,7 @@ public abstract class InputFormatBase<K,
    * @see #setRanges(Job, Collection)
    */
   protected static List<Range> getRanges(JobContext context) throws IOException {
-    return InputConfigurator.getRanges(CLASS, context.getConfiguration());
+    return InputConfigurator.getRanges(CLASS, getConfiguration(context));
   }
   
   /**
@@ -341,7 +342,7 @@ public abstract class InputFormatBase<K,
    * @see #fetchColumns(Job, Collection)
    */
   protected static Set<Pair<Text,Text>> getFetchedColumns(JobContext context)
{
-    return InputConfigurator.getFetchedColumns(CLASS, context.getConfiguration());
+    return InputConfigurator.getFetchedColumns(CLASS, getConfiguration(context));
   }
   
   /**
@@ -367,7 +368,7 @@ public abstract class InputFormatBase<K,
    * @see #addIterator(Job, IteratorSetting)
    */
   protected static List<IteratorSetting> getIterators(JobContext context) {
-    return InputConfigurator.getIterators(CLASS, context.getConfiguration());
+    return InputConfigurator.getIterators(CLASS, getConfiguration(context));
   }
   
   /**
@@ -398,7 +399,7 @@ public abstract class InputFormatBase<K,
    * @see #setAutoAdjustRanges(Job, boolean)
    */
   protected static boolean getAutoAdjustRanges(JobContext context) {
-    return InputConfigurator.getAutoAdjustRanges(CLASS, context.getConfiguration());
+    return InputConfigurator.getAutoAdjustRanges(CLASS, getConfiguration(context));
   }
   
   /**
@@ -427,7 +428,7 @@ public abstract class InputFormatBase<K,
    * @see #setScanIsolation(Job, boolean)
    */
   protected static boolean isIsolated(JobContext context) {
-    return InputConfigurator.isIsolated(CLASS, context.getConfiguration());
+    return InputConfigurator.isIsolated(CLASS, getConfiguration(context));
   }
   
   /**
@@ -457,7 +458,7 @@ public abstract class InputFormatBase<K,
    * @see #setLocalIterators(Job, boolean)
    */
   protected static boolean usesLocalIterators(JobContext context) {
-    return InputConfigurator.usesLocalIterators(CLASS, context.getConfiguration());
+    return InputConfigurator.usesLocalIterators(CLASS, getConfiguration(context));
   }
   
   /**
@@ -505,7 +506,7 @@ public abstract class InputFormatBase<K,
    * @see #setOfflineTableScan(Job, boolean)
    */
   protected static boolean isOfflineScan(JobContext context) {
-    return InputConfigurator.isOfflineScan(CLASS, context.getConfiguration());
+    return InputConfigurator.isOfflineScan(CLASS, getConfiguration(context));
   }
   
   /**
@@ -519,7 +520,7 @@ public abstract class InputFormatBase<K,
    * @since 1.5.0
    */
   protected static TabletLocator getTabletLocator(JobContext context) throws TableNotFoundException
{
-    return InputConfigurator.getTabletLocator(CLASS, context.getConfiguration());
+    return InputConfigurator.getTabletLocator(CLASS, getConfiguration(context));
   }
   
   // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext
job)
@@ -533,7 +534,7 @@ public abstract class InputFormatBase<K,
    * @since 1.5.0
    */
   protected static void validateOptions(JobContext context) throws IOException {
-    InputConfigurator.validateOptions(CLASS, context.getConfiguration());
+    InputConfigurator.validateOptions(CLASS, getConfiguration(context));
   }
   
   /**
@@ -1341,5 +1342,17 @@ public abstract class InputFormatBase<K,
     }
     
   }
+
+  // use reflection to pull the Configuration out of the JobContext for Hadoop 1 and Hadoop
2 compatibility
+  public static Configuration getConfiguration(JobContext context) {
+    try {
+      Class c = InputFormatBase.class.getClassLoader().loadClass("org.apache.hadoop.mapreduce.JobContext");
+      Method m = c.getMethod("getConfiguration");
+      Object o = m.invoke(context, new Object[0]);
+      return (Configuration)o;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
   
 }



Mime
View raw message