hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r399500 - in /lucene/hadoop/trunk: ./ bin/ src/java/org/apache/hadoop/conf/ src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/io/ src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/util/
Date Thu, 04 May 2006 00:13:33 GMT
Author: cutting
Date: Wed May  3 17:13:31 2006
New Revision: 399500

URL: http://svn.apache.org/viewcvs?rev=399500&view=rev
Log:
Better fix for HADOOP-189.  Tested in both distributed and standalone mode.  Note that WritableFactories
is not quite functioning correctly and there are a few hacks around this that should be removed
when that's better understood.

Added:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java
      - copied, changed from r399426, lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java
Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/bin/hadoop
    lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ObjectWritable.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableName.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/CHANGES.txt?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed May  3 17:13:31 2006
@@ -167,6 +167,11 @@
     'bin/hadoop jar build/hadoop-streaming.jar' to get details.
     (Michel Tourn via cutting)
 
+44. HADOOP-189.  Fix MapReduce in standalone configuration to
+    correctly handle job jar files that contain a lib directory with
+    nested jar files.  (cutting)
+
+
 Release 0.1.1 - 2006-04-08
 
  1. Added CHANGES.txt, logging all significant changes to Hadoop.  (cutting)

Modified: lucene/hadoop/trunk/bin/hadoop
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/bin/hadoop?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/bin/hadoop (original)
+++ lucene/hadoop/trunk/bin/hadoop Wed May  3 17:13:31 2006
@@ -135,14 +135,7 @@
 elif [ "$COMMAND" = "job" ] ; then
   CLASS=org.apache.hadoop.mapred.JobClient
 elif [ "$COMMAND" = "jar" ] ; then
-  JAR="$1"
-  shift
-  CLASS=`"$0" org.apache.hadoop.util.PrintJarMainClass "$JAR"`
-  if [ $? != 0 ]; then
-    echo "Error: Could not find main class in jar file $JAR"
-    exit 1
-  fi
-  CLASSPATH=${CLASSPATH}:${JAR}
+  CLASS=org.apache.hadoop.util.RunJar
 else
   CLASS=$COMMAND
 fi

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java Wed May  3 17:13:31
2006
@@ -241,7 +241,7 @@
     if (valueString == null)
       return defaultValue;
     try {
-      return Class.forName(valueString);
+      return classLoader.loadClass(valueString);
     } catch (ClassNotFoundException e) {
       throw new RuntimeException(e);
     }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java Wed May  3 17:13:31
2006
@@ -49,6 +49,9 @@
     Daemon leaseChecker;
     private Configuration conf;
     
+    // required for unknown reason to make WritableFactories work distributed
+    static { new DFSFileInfo(); }
+
     /**
      * A map from name -> DFSOutputStream of files that are currently being
      * written by this client.

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java Wed May  3 17:13:31 2006
@@ -65,6 +65,9 @@
     //private static long numGigs = Configuration.get().getLong("dfs.datanode.maxgigs", 100);
     //
 
+    // required for unknown reason to make WritableFactories work distributed
+    static { new BlockCommand(); }
+
     /**
      * Util method to build socket addr from string
      */

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ObjectWritable.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ObjectWritable.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ObjectWritable.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ObjectWritable.java Wed May  3 17:13:31
2006
@@ -88,7 +88,8 @@
       declaredClass = (Class)PRIMITIVE_NAMES.get(className);
       if (declaredClass == null) {
         try {
-          declaredClass = Class.forName(className);
+          declaredClass =
+            Thread.currentThread().getContextClassLoader().loadClass(className);
         } catch (ClassNotFoundException e) {
           throw new RuntimeException(e.toString());
         }
@@ -170,7 +171,8 @@
     Class declaredClass = (Class)PRIMITIVE_NAMES.get(className);
     if (declaredClass == null) {
       try {
-        declaredClass = Class.forName(className);
+        declaredClass =
+          Thread.currentThread().getContextClassLoader().loadClass(className);
       } catch (ClassNotFoundException e) {
         throw new RuntimeException(e.toString());
       }
@@ -215,7 +217,8 @@
     } else {                                      // Writable
       Class instanceClass = null;
       try {
-        instanceClass = Class.forName(UTF8.readString(in));
+        instanceClass = Thread.currentThread().getContextClassLoader()
+          .loadClass(UTF8.readString(in));
       } catch (ClassNotFoundException e) {
         throw new RuntimeException(e.toString());
       }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableName.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableName.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableName.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableName.java Wed May  3 17:13:31
2006
@@ -62,7 +62,7 @@
     if (writableClass != null)
       return writableClass;
     try {
-      return Class.forName(name);
+      return Thread.currentThread().getContextClassLoader().loadClass(name);
     } catch (ClassNotFoundException e) {
       throw new IOException(e.toString());
     }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java Wed May  3 17:13:31
2006
@@ -38,6 +38,9 @@
 
     static long MAX_JOBPROFILE_AGE = 1000 * 2;
 
+    // required for unknown reason to make WritableFactories work distributed
+    static { new JobStatus(); new JobProfile(); }
+
     /**
      * A NetworkedJob is an implementation of RunningJob.  It holds
      * a JobProfile object to provide some info, and interacts with the

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Wed May  3 17:13:31
2006
@@ -49,6 +49,10 @@
       idFormat.setMinimumIntegerDigits(4);
       idFormat.setGroupingUsed(false);
     }
+
+    // required for unknown reason to make WritableFactories work distributed
+    static { new TaskTrackerStatus(); }
+
     private int nextJobId = 1;
 
     public static final Logger LOG = LogFormatter.getLogger("org.apache.hadoop.mapred.JobTracker");

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java Wed May  3 17:13:31
2006
@@ -18,6 +18,7 @@
 import org.apache.hadoop.conf.*;
 import org.apache.hadoop.util.LogFormatter;
 import org.apache.hadoop.fs.*;
+import org.apache.hadoop.util.*;
 
 import java.io.*;
 import java.util.jar.*;
@@ -78,7 +79,7 @@
 
       String jar = conf.getJar();
       if (jar != null) {                      // if jar exists, it into workDir
-        unJar(new File(jar), workDir);
+        RunJar.unJar(new File(jar), workDir);
         File[] libs = new File(workDir, "lib").listFiles();
         if (libs != null) {
           for (int i = 0; i < libs.length; i++) {
@@ -220,37 +221,6 @@
         
     }
     return text;
-  }
-
-  private void unJar(File jarFile, File toDir) throws IOException {
-    JarFile jar = new JarFile(jarFile);
-    try {
-      Enumeration entries = jar.entries();
-      while (entries.hasMoreElements()) {
-        JarEntry entry = (JarEntry)entries.nextElement();
-        if (!entry.isDirectory()) {
-          InputStream in = jar.getInputStream(entry);
-          try {
-            File file = new File(toDir, entry.getName());
-            file.getParentFile().mkdirs();
-            OutputStream out = new FileOutputStream(file);
-            try {
-              byte[] buffer = new byte[8192];
-              int i;
-              while ((i = in.read(buffer)) != -1) {
-                out.write(buffer, 0, i);
-              }
-            } finally {
-              out.close();
-            }
-          } finally {
-            in.close();
-          }
-        }
-      }
-    } finally {
-      jar.close();
-    }
   }
 
   /**

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Wed May  3 17:13:31
2006
@@ -38,6 +38,9 @@
 
     static final int STALE_STATE = 1;
 
+    // required for unknown reason to make WritableFactories work distributed
+    static { new MapTask(); new ReduceTask(); new MapOutputLocation(); }
+
     public static final Logger LOG =
     LogFormatter.getLogger("org.apache.hadoop.mapred.TaskTracker");
 

Copied: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java (from r399426, lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java)
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java?p2=lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java&p1=lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java&r1=399426&r2=399500&rev=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java Wed May  3 17:13:31 2006
@@ -107,14 +107,15 @@
     unJar(file, workDir);
     
     ArrayList classPath = new ArrayList();
+    classPath.add(new File(workDir+"/").toURL());
+    classPath.add(file.toURL());
+    classPath.add(new File(workDir, "classes/").toURL());
     File[] libs = new File(workDir, "lib").listFiles();
     if (libs != null) {
       for (int i = 0; i < libs.length; i++) {
         classPath.add(libs[i].toURL());
       }
     }
-    classPath.add(new File(workDir, "classes/").toURL());
-    classPath.add(new File(workDir+"/").toURL());
     ClassLoader loader =
       new URLClassLoader((URL[])classPath.toArray(new URL[0]));
 



Mime
View raw message