incubator-bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject svn commit: r1178673 - in /incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src: common/hadoop/patch deb/hadoop/rules rpm/hadoop/SPECS/hadoop.spec
Date Tue, 04 Oct 2011 02:08:03 GMT
Author: rvs
Date: Tue Oct  4 02:08:03 2011
New Revision: 1178673

URL: http://svn.apache.org/viewvc?rev=1178673&view=rev
Log:
MAPREDUCE-3138

Added:
    incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/common/hadoop/patch
Modified:
    incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/deb/hadoop/rules
    incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec

Added: incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/common/hadoop/patch
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/common/hadoop/patch?rev=1178673&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/common/hadoop/patch (added)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/common/hadoop/patch Tue Oct
 4 02:08:03 2011
@@ -0,0 +1,247 @@
+diff --git mapreduce/src/java/org/apache/hadoop/mapreduce/ContextFactory.java mapreduce/src/java/org/apache/hadoop/mapreduce/ContextFactory.java
+new file mode 100644
+index 0000000..1b1a85b
+--- /dev/null
++++ mapreduce/src/java/org/apache/hadoop/mapreduce/ContextFactory.java
+@@ -0,0 +1,241 @@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements.  See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership.  The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License.  You may obtain a copy of the License at
++ *
++ *     http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.hadoop.mapreduce;
++ 
++import java.io.IOException;
++import java.lang.reflect.Constructor;
++import java.lang.reflect.Field;
++import java.lang.reflect.InvocationTargetException;
++
++import org.apache.hadoop.conf.Configuration;
++ 
++/**
++ * A factory to allow applications to deal with inconsistencies between
++ * MapReduce Context Objects API between hadoop-0.20 and later versions.
++ */
++public class ContextFactory {
++
++  private static final Constructor<?> JOB_CONTEXT_CONSTRUCTOR;
++  private static final Constructor<?> TASK_CONTEXT_CONSTRUCTOR;
++  private static final Constructor<?> MAP_CONTEXT_CONSTRUCTOR;
++  private static final Constructor<?> MAP_CONTEXT_IMPL_CONSTRUCTOR;
++  private static final boolean useV21;
++  
++  private static final Field REPORTER_FIELD;
++  private static final Field READER_FIELD;
++  private static final Field WRITER_FIELD;
++  private static final Field OUTER_MAP_FIELD;
++  private static final Field WRAPPED_CONTEXT_FIELD;
++  
++  static {
++    boolean v21 = true;
++    final String PACKAGE = "org.apache.hadoop.mapreduce";
++    try {
++      Class.forName(PACKAGE + ".task.JobContextImpl");
++    } catch (ClassNotFoundException cnfe) {
++      v21 = false;
++    }
++    useV21 = v21;
++    Class<?> jobContextCls;
++    Class<?> taskContextCls;
++    Class<?> taskIOContextCls;
++    Class<?> mapCls;
++    Class<?> mapContextCls;
++    Class<?> innerMapContextCls;
++    try {
++      if (v21) {
++        jobContextCls = 
++          Class.forName(PACKAGE+".task.JobContextImpl");
++        taskContextCls = 
++          Class.forName(PACKAGE+".task.TaskAttemptContextImpl");
++        taskIOContextCls = 
++          Class.forName(PACKAGE+".task.TaskInputOutputContextImpl");
++        mapContextCls = Class.forName(PACKAGE + ".task.MapContextImpl");
++        mapCls = Class.forName(PACKAGE + ".lib.map.WrappedMapper");
++        innerMapContextCls = 
++          Class.forName(PACKAGE+".lib.map.WrappedMapper$Context");
++      } else {
++        jobContextCls = 
++          Class.forName(PACKAGE+".JobContext");
++        taskContextCls = 
++          Class.forName(PACKAGE+".TaskAttemptContext");
++        taskIOContextCls = 
++          Class.forName(PACKAGE+".TaskInputOutputContext");
++        mapContextCls = Class.forName(PACKAGE + ".MapContext");
++        mapCls = Class.forName(PACKAGE + ".Mapper");
++        innerMapContextCls = 
++          Class.forName(PACKAGE+".Mapper$Context");      
++      }
++    } catch (ClassNotFoundException e) {
++      throw new IllegalArgumentException("Can't find class", e);
++    }
++    try {
++      JOB_CONTEXT_CONSTRUCTOR = 
++        jobContextCls.getConstructor(Configuration.class, JobID.class);
++      JOB_CONTEXT_CONSTRUCTOR.setAccessible(true);
++      TASK_CONTEXT_CONSTRUCTOR = 
++        taskContextCls.getConstructor(Configuration.class, 
++                                      TaskAttemptID.class);
++      TASK_CONTEXT_CONSTRUCTOR.setAccessible(true);
++      if (useV21) {
++        MAP_CONTEXT_CONSTRUCTOR = 
++          innerMapContextCls.getConstructor(mapCls,
++                                            MapContext.class);
++        MAP_CONTEXT_IMPL_CONSTRUCTOR =
++          mapContextCls.getDeclaredConstructor(Configuration.class, 
++                                               TaskAttemptID.class,
++                                               RecordReader.class,
++                                               RecordWriter.class,
++                                               OutputCommitter.class,
++                                               StatusReporter.class,
++                                               InputSplit.class);
++        MAP_CONTEXT_IMPL_CONSTRUCTOR.setAccessible(true);
++        WRAPPED_CONTEXT_FIELD = 
++          innerMapContextCls.getDeclaredField("mapContext");
++        WRAPPED_CONTEXT_FIELD.setAccessible(true);
++      } else {
++        MAP_CONTEXT_CONSTRUCTOR = 
++          innerMapContextCls.getConstructor(mapCls,
++                                            Configuration.class, 
++                                            TaskAttemptID.class,
++                                            RecordReader.class,
++                                            RecordWriter.class,
++                                            OutputCommitter.class,
++                                            StatusReporter.class,
++                                            InputSplit.class);
++        MAP_CONTEXT_IMPL_CONSTRUCTOR = null;
++        WRAPPED_CONTEXT_FIELD = null;
++      }
++      MAP_CONTEXT_CONSTRUCTOR.setAccessible(true);
++      REPORTER_FIELD = taskIOContextCls.getDeclaredField("reporter");
++      REPORTER_FIELD.setAccessible(true);
++      READER_FIELD = mapContextCls.getDeclaredField("reader");
++      READER_FIELD.setAccessible(true);
++      WRITER_FIELD = taskIOContextCls.getDeclaredField("output");
++      WRITER_FIELD.setAccessible(true);
++      OUTER_MAP_FIELD = innerMapContextCls.getDeclaredField("this$0");
++      OUTER_MAP_FIELD.setAccessible(true);
++    } catch (SecurityException e) {
++      throw new IllegalArgumentException("Can't run constructor ", e);
++    } catch (NoSuchMethodException e) {
++      throw new IllegalArgumentException("Can't find constructor ", e);
++    } catch (NoSuchFieldException e) {
++      throw new IllegalArgumentException("Can't find field ", e);
++    }
++  }
++
++  /**
++   * Clone a job or task attempt context with a new configuration.
++   * @param original the original context
++   * @param conf the new configuration
++   * @return a new context object
++   * @throws InterruptedException 
++   * @throws IOException 
++   */
++  @SuppressWarnings("unchecked")
++  public static JobContext cloneContext(JobContext original,
++                                        Configuration conf
++                                        ) throws IOException, 
++                                                 InterruptedException {
++    try {
++      if (original instanceof MapContext<?,?,?,?>) {
++        return cloneMapContext((Mapper.Context) original, conf, null, null);        
++      } else if (original instanceof ReduceContext<?,?,?,?>) {
++        throw new IllegalArgumentException("can't clone ReduceContext");
++      } else if (original instanceof TaskAttemptContext) {
++        TaskAttemptContext spec = (TaskAttemptContext) original;
++        return (JobContext) 
++          TASK_CONTEXT_CONSTRUCTOR.newInstance(conf, spec.getTaskAttemptID());
++      } else {
++        return (JobContext) 
++        JOB_CONTEXT_CONSTRUCTOR.newInstance(conf, original.getJobID());           
++      }
++    } catch (InstantiationException e) {
++      throw new IllegalArgumentException("Can't clone object", e);
++    } catch (IllegalAccessException e) {
++      throw new IllegalArgumentException("Can't clone object", e);
++    } catch (InvocationTargetException e) {
++      throw new IllegalArgumentException("Can't clone object", e);
++    }
++  }
++  
++  /**
++   * Copy a mapper context, optionally replacing the input and output.
++   * @param <K1> input key type
++   * @param <V1> input value type
++   * @param <K2> output key type
++   * @param <V2> output value type
++   * @param context the context to clone
++   * @param conf a new configuration
++   * @param reader Reader to read from. Null means to clone from context.
++   * @param writer Writer to write to. Null means to clone from context.
++   * @return a new context. it will not be the same class as the original.
++   * @throws IOException
++   * @throws InterruptedException
++   */
++  @SuppressWarnings("unchecked")
++  public static <K1,V1,K2,V2> Mapper<K1,V1,K2,V2>.Context 
++       cloneMapContext(MapContext<K1,V1,K2,V2> context,
++                       Configuration conf,
++                       RecordReader<K1,V1> reader,
++                       RecordWriter<K2,V2> writer
++                      ) throws IOException, InterruptedException {
++    try {
++      // get the outer object pointer
++      Object outer = OUTER_MAP_FIELD.get(context);
++      // if it is a wrapped 21 context, unwrap it
++      if ("org.apache.hadoop.mapreduce.lib.map.WrappedMapper$Context".equals
++            (context.getClass().getName())) {
++        context = (MapContext<K1,V1,K2,V2>) WRAPPED_CONTEXT_FIELD.get(context);
++      }
++      // if the reader or writer aren't given, use the same ones
++      if (reader == null) {
++        reader = (RecordReader<K1,V1>) READER_FIELD.get(context);
++      }
++      if (writer == null) {
++        writer = (RecordWriter<K2,V2>) WRITER_FIELD.get(context);
++      }
++      if (useV21) {
++        Object basis = 
++          MAP_CONTEXT_IMPL_CONSTRUCTOR.newInstance(conf, 
++                                                   context.getTaskAttemptID(),
++                                                   reader, writer,
++                                                   context.getOutputCommitter(),
++                                                   REPORTER_FIELD.get(context),
++                                                   context.getInputSplit());
++        return (Mapper.Context) 
++          MAP_CONTEXT_CONSTRUCTOR.newInstance(outer, basis);
++      } else {
++        return (Mapper.Context)
++          MAP_CONTEXT_CONSTRUCTOR.newInstance(outer,
++                                              conf, context.getTaskAttemptID(),
++                                              reader, writer,
++                                              context.getOutputCommitter(),
++                                              REPORTER_FIELD.get(context),
++                                              context.getInputSplit());
++      }
++    } catch (IllegalAccessException e) {
++      throw new IllegalArgumentException("Can't access field", e);
++    } catch (InstantiationException e) {
++      throw new IllegalArgumentException("Can't create object", e);
++    } catch (InvocationTargetException e) {
++      throw new IllegalArgumentException("Can't invoke constructor", e);
++    }
++  }
++}

Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/deb/hadoop/rules
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/deb/hadoop/rules?rev=1178673&r1=1178672&r2=1178673&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/deb/hadoop/rules (original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/deb/hadoop/rules Tue Oct  4
02:08:03 2011
@@ -39,6 +39,7 @@ endif
 build: build-stamp
 build-stamp:
 	dh_testdir
+	patch -p0 < debian/patch
 	env HADOOP_VERSION=${hadoop_version} \
 		sh debian/do-component-build -Divy.home=`pwd`/debian/.ivy
 	touch $@

Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec?rev=1178673&r1=1178672&r2=1178673&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
Tue Oct  4 02:08:03 2011
@@ -107,6 +107,7 @@ Source5: hadoop-init.tmpl.suse
 Source6: hadoop.1
 Source7: hadoop-fuse-dfs.1
 Source8: hadoop-fuse.default
+Patch0: patch
 Buildroot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 BuildRequires: python >= 2.4, git, fuse-devel,fuse, automake, autoconf
 Requires: textutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service
@@ -293,6 +294,7 @@ before continuing operation.
 
 %prep
 %setup -n apache-hadoop-common-e262faf
+%patch0 -p0
 
 %build
 # This assumes that you installed Java JDK 6 and set JAVA_HOME



Mime
View raw message