hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r608050 - in /lucene/hadoop/trunk: ./ src/java/org/apache/hadoop/fs/ src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/security/ src/java/org/apache/hadoop/util/ src/test/org/apache/hadoop/dfs/ src/test/org/apache/hadoop/fs/
Date Wed, 02 Jan 2008 09:19:14 GMT
Author: acmurthy
Date: Wed Jan  2 01:19:11 2008
New Revision: 608050

URL: http://svn.apache.org/viewvc?rev=608050&view=rev
Log:
HADOOP-2344. Enhance the utility for executing shell commands to read the
stdout/stderr streams while waiting for the command to finish (to free up
the buffers). Also, this patch throws away stderr of the DF utility.
@deprecated
  org.apache.hadoop.fs.ShellCommand for org.apache.hadoop.util.Shell
  org.apache.hadoop.util.ShellUtil for org.apache.hadoop.util.Shell.ShellCommandExecutor

Added:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/util/Shell.java
Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/DF.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/DU.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileUtil.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/ShellCommand.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/security/UnixUserGroupInformation.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ShellUtil.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalDirAllocator.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystemPermission.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=608050&r1=608049&r2=608050&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed Jan  2 01:19:11 2008
@@ -306,6 +306,14 @@
     HADOOP-2492. Additional debugging in the rpc server to better 
     diagnose ConcurrentModificationException. (dhruba)
 
+    HADOOP-2344. Enhance the utility for executing shell commands to read the
+    stdout/stderr streams while waiting for the command to finish (to free up
+    the buffers). Also, this patch throws away stderr of the DF utility.
+    @deprecated 
+      org.apache.hadoop.fs.ShellCommand for org.apache.hadoop.util.Shell
+      org.apache.hadoop.util.ShellUtil for 
+        org.apache.hadoop.util.Shell.ShellCommandExecutor
+
 Release 0.15.2 - 2008-01-02
 
   BUG FIXES

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/DF.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/DF.java?rev=608050&r1=608049&r2=608050&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/DF.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/DF.java Wed Jan  2 01:19:11 2008
@@ -24,10 +24,11 @@
 import java.util.StringTokenizer;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Shell;
 
 /** Filesystem disk space usage statistics.  Uses the unix 'df' program.
  * Tested on Linux, FreeBSD, Cygwin. */
-public class DF extends ShellCommand {
+public class DF extends Shell {
   public static final long DF_INTERVAL_DEFAULT = 3 * 1000; // default DF refresh interval

   
   private String  dirPath;
@@ -95,7 +96,9 @@
   }
 
   protected String[] getExecString() {
-    return new String[] {"df","-k", dirPath};
+    // ignoring the error since the exit code it enough
+    return new String[] {"bash","-c","exec 'df' '-k' '" + dirPath 
+                         + "' 2>/dev/null"};
   }
   
   protected void parseExecResult(BufferedReader lines) throws IOException {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/DU.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/DU.java?rev=608050&r1=608049&r2=608050&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/DU.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/DU.java Wed Jan  2 01:19:11 2008
@@ -23,9 +23,10 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.dfs.FSConstants;
+import org.apache.hadoop.util.Shell;
 
 /** Filesystem disk space usage statistics.  Uses the unix 'du' program*/
-public class DU extends ShellCommand {
+public class DU extends Shell {
   private String  dirPath;
 
   private long used;

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileUtil.java?rev=608050&r1=608049&r2=608050&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileUtil.java Wed Jan  2 01:19:11 2008
@@ -25,6 +25,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Shell;
 
 /**
  * A collection of file-processing util methods
@@ -257,7 +258,7 @@
   /**
    * This class is only used on windows to invoke the cygpath command.
    */
-  private static class CygPathCommand extends ShellCommand {
+  private static class CygPathCommand extends Shell {
     String[] command;
     String result;
     CygPathCommand(String path) throws IOException {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=608050&r1=608049&r2=608050&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java Wed Jan  2 01:19:11
2008
@@ -28,6 +28,7 @@
 import org.apache.hadoop.fs.permission.*;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Shell;
 
 /****************************************************************
  * Implement the FileSystem API for the raw local filesystem.
@@ -399,7 +400,7 @@
       try {
         StringTokenizer t = new StringTokenizer(
             execCommand(new File(getPath().toUri()), 
-                        ShellCommand.getGET_PERMISSION_COMMAND()));
+                        Shell.getGET_PERMISSION_COMMAND()));
         //expected format
         //-rw-------    1 username groupname ...
         String permission = t.nextToken();
@@ -449,7 +450,7 @@
     //[OWNER][:[GROUP]]
     String s = (username == null? "": username)
              + (groupname == null? "": ":" + groupname);
-    execCommand(pathToFile(p), ShellCommand.SET_OWNER_COMMAND, s);
+    execCommand(pathToFile(p), Shell.SET_OWNER_COMMAND, s);
   }
 
   /**
@@ -458,7 +459,7 @@
   @Override
   public void setPermission(Path p, FsPermission permission
       ) throws IOException {
-    execCommand(pathToFile(p), ShellCommand.SET_PERMISSION_COMMAND,
+    execCommand(pathToFile(p), Shell.SET_PERMISSION_COMMAND,
         String.format("%04o", permission.toShort()));
   }
 
@@ -466,7 +467,7 @@
     String[] args = new String[cmd.length + 1];
     System.arraycopy(cmd, 0, args, 0, cmd.length);
     args[cmd.length] = f.getCanonicalPath();
-    String output = ShellCommand.execCommand(args);
+    String output = Shell.execCommand(args);
     return output;
   }
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/ShellCommand.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/ShellCommand.java?rev=608050&r1=608049&r2=608050&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/ShellCommand.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/ShellCommand.java Wed Jan  2 01:19:11
2008
@@ -17,111 +17,9 @@
  */
 package org.apache.hadoop.fs;
 
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.BufferedReader;
+import org.apache.hadoop.util.Shell;
 
-/** A base class for running a unix command like du or df*/
-abstract public class ShellCommand {
-  /** a Unix command to get the current user's name */
-  public final static String USER_NAME_COMMAND = "whoami";
-  /** a Unix command to get the current user's groups list */
-  public static String[] getGROUPS_COMMAND() {
-    return new String[]{"bash", "-c", "groups"};
-  }
-  /** a Unix command to set permission */
-  public static final String SET_PERMISSION_COMMAND = "chmod";
-  /** a Unix command to set owner */
-  public static final String SET_OWNER_COMMAND = "chown";
-  /** Return a Unix command to get permission information. */
-  public static String[] getGET_PERMISSION_COMMAND() {
-    return new String[]{"ls", "-ld"};
-  }
-
-  private long    interval;   // refresh interval in msec
-  private long    lastTime;   // last time the command was performed
-  
-  ShellCommand() {
-    this(0L);
-  }
-  
-  ShellCommand( long interval ) {
-    this.interval = interval;
-    this.lastTime = (interval<0) ? 0 : -interval;
-  }
-  
-  /** check to see if a command needs to be execuated */
-  protected void run() throws IOException {
-    if (lastTime + interval > System.currentTimeMillis())
-      return;
-    runCommand();
-  }
-
-  /** Run a command */
-  private void runCommand() throws IOException { 
-    Process process;
-    process = Runtime.getRuntime().exec(getExecString());
-
-    try {
-      if (process.waitFor() != 0) {
-        throw new IOException
-          (new BufferedReader(new InputStreamReader(process.getErrorStream()))
-           .readLine());
-      }
-      parseExecResult(new BufferedReader(
-          new InputStreamReader(process.getInputStream())));
-    } catch (InterruptedException e) {
-      throw new IOException(e.toString());
-    } finally {
-      process.destroy();
-      lastTime = System.currentTimeMillis();
-    }
-  }
-
-  /** return an array comtaining the command name & its parameters */ 
-  protected abstract String[] getExecString();
-  
-  /** Parse the execution result */
-  protected abstract void parseExecResult(BufferedReader lines)
-  throws IOException;
-
-  /// A simple implementation of Command
-  private static class SimpleCommandExecutor extends ShellCommand {
-    
-    private String[] command;
-    private StringBuffer reply;
-    
-    SimpleCommandExecutor(String[] execString) {
-      command = execString;
-    }
-
-    @Override
-    protected String[] getExecString() {
-      return command;
-    }
-
-    @Override
-    protected void parseExecResult(BufferedReader lines) throws IOException {
-      reply = new StringBuffer();
-      char[] buf = new char[512];
-      int nRead;
-      while ( (nRead = lines.read(buf, 0, buf.length)) > 0 ) {
-        reply.append(buf, 0, nRead);
-      }
-    }
-    
-    String getReply() {
-      return (reply == null) ? "" : reply.toString();
-    }
-  }
-  
-  /** 
-   * Static method to execute a command. Covers most of the simple cases 
-   * without requiring the user to implement Command interface.
-   */
-  public static String execCommand(String ... cmd) throws IOException {
-    SimpleCommandExecutor exec = new SimpleCommandExecutor(cmd);
-    exec.run();
-    return exec.getReply();
-  }
-}
+/** A base class for running a unix command like du or df.
+ * @deprecated Use {@link Shell} instead.
+ */
+abstract public class ShellCommand extends Shell {}

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java?rev=608050&r1=608049&r2=608050&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java Wed Jan  2 01:19:11
2008
@@ -20,6 +20,7 @@
 import org.apache.commons.logging.*;
 
 import org.apache.hadoop.fs.*;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.filecache.*;
 import org.apache.hadoop.util.*;
@@ -38,7 +39,7 @@
     LogFactory.getLog("org.apache.hadoop.mapred.TaskRunner");
 
   volatile boolean killed = false;
-  private Process process;
+  private ShellCommandExecutor shexec; // shell terminal for running the task
   private Task t;
   private TaskTracker tracker;
 
@@ -440,57 +441,27 @@
   }
 
   /**
-   * Append the contents of the input stream to the output file. Both streams 
-   * are closed upon exit.
-   * @param in the stream to read
-   * @param outName the filename to append the data to
-   * @throws IOException if something goes wrong
-   */
-  private void copyStream(InputStream in, File outName) throws IOException {
-    try {
-      OutputStream out = new FileOutputStream(outName, true);
-      try {
-        byte[] buffer = new byte[1024];
-        int len = in.read(buffer);
-        while (len > 0) {
-          out.write(buffer, 0, len);
-          len = in.read(buffer);
-        }
-      } finally {
-        out.close();
-      }
-    } finally {
-      in.close();
-    }
-  }
-
-  /**
    * Run the child process
    */
   private void runChild(List<String> args, File dir,
                         String taskid) throws IOException {
 
     try {
-      ShellUtil shexec = new ShellUtil(args, dir, System.getenv());
+      shexec = new ShellCommandExecutor(args.toArray(new String[0]), dir);
       shexec.execute();
-      process = shexec.getProcess();
+    } catch (IOException ioe) {
+      // do nothing
+      // error and output are appropriately redirected
+    } finally { // handle the exit code
       int exit_code = shexec.getExitCode();
      
       if (!killed && exit_code != 0) {
         if (exit_code == 65) {
           tracker.getTaskTrackerMetrics().taskFailedPing();
         }
-        copyStream(process.getInputStream(), 
-                   TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDOUT));
-        copyStream(process.getErrorStream(), 
-                   TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDERR));
         throw new IOException("Task process exit with nonzero status of " +
                               exit_code + ".");
       }
-    } catch (InterruptedException e) {
-      throw new IOException(e.toString());
-    } finally {
-      kill();      
     }
   }
 
@@ -498,8 +469,11 @@
    * Kill the child process
    */
   public void kill() {
-    if (process != null) {
-      process.destroy();
+    if (shexec != null) {
+      Process process = shexec.getProcess();
+      if (process != null) {
+        process.destroy();
+      }
     }
     killed = true;
   }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=608050&r1=608049&r2=608050&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Wed Jan  2 01:19:11
2008
@@ -58,6 +58,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RemoteException;
@@ -75,7 +76,6 @@
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.RunJar;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.ShellUtil;
 import org.apache.hadoop.util.DiskChecker.DiskErrorException;
 import org.apache.log4j.LogManager;
 
@@ -1637,22 +1637,13 @@
      * @throws IOException
      */
     public void runScript(List<String> args, File dir) throws IOException {
-      Process process = null;
-      try {
-        ShellUtil shexec = new ShellUtil(args, dir, System.getenv());
-        shexec.execute();
-        process = shexec.getProcess();
-        int exit_code = shexec.getExitCode();
-        if (exit_code != 0) {
-          throw new IOException("Task debug script exit with nonzero "
-                                +"status of " + exit_code + ".");
-        }
-      } catch (InterruptedException e) {
-          throw new IOException(e.toString());
-      } finally {
-        if (process != null) {
-          process.destroy();
-        }
+      ShellCommandExecutor shexec = 
+              new ShellCommandExecutor(args.toArray(new String[0]), dir);
+      shexec.execute();
+      int exitCode = shexec.getExitCode();
+      if (exitCode != 0) {
+        throw new IOException("Task debug script exit with nonzero status of " 
+                              + exitCode + ".");
       }
     }
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/security/UnixUserGroupInformation.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/security/UnixUserGroupInformation.java?rev=608050&r1=608049&r2=608050&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/security/UnixUserGroupInformation.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/security/UnixUserGroupInformation.java
Wed Jan  2 01:19:11 2008
@@ -29,7 +29,7 @@
 import javax.security.auth.login.LoginException;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.ShellCommand;
+import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.WritableUtils;
 
@@ -289,10 +289,10 @@
    */
   static String getUnixUserName() throws IOException {
     String[] result = executeShellCommand(
-        new String[]{ShellCommand.USER_NAME_COMMAND});
+        new String[]{Shell.USER_NAME_COMMAND});
     if (result.length!=1) {
       throw new IOException("Expect one token as the result of " + 
-          ShellCommand.USER_NAME_COMMAND + ": " + toString(result));
+          Shell.USER_NAME_COMMAND + ": " + toString(result));
     }
     return result[0];
   }
@@ -303,13 +303,13 @@
    * @throws IOException if encounter any error when running the command
    */
   private static String[] getUnixGroups() throws IOException {
-    return executeShellCommand(ShellCommand.getGROUPS_COMMAND());
+    return executeShellCommand(Shell.getGROUPS_COMMAND());
   }
   
   /* Execute a command and return the result as an array of Strings */
   private static String[] executeShellCommand(String[] command)
   throws IOException {
-    String groups = ShellCommand.execCommand(command);
+    String groups = Shell.execCommand(command);
     StringTokenizer tokenizer = new StringTokenizer(groups);
     int numOfTokens = tokenizer.countTokens();
     String[] tokens = new String[numOfTokens];

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/Shell.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/Shell.java?rev=608050&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/Shell.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/Shell.java Wed Jan  2 01:19:11 2008
@@ -0,0 +1,268 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import java.util.Map;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.BufferedReader;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/** 
+ * A base class for running a Unix command.
+ * 
+ * <code>Shell</code> can be used to run unix commands like <code>du</code>
or
+ * <code>df</code>. It also offers facilities to gate commands by 
+ * time-intervals.
+ */
+abstract public class Shell {
+  
+  public static final Log LOG = LogFactory.getLog(Shell.class);
+  
+  /** a Unix command to get the current user's name */
+  public final static String USER_NAME_COMMAND = "whoami";
+  /** a Unix command to get the current user's groups list */
+  public static String[] getGROUPS_COMMAND() {
+    return new String[]{"bash", "-c", "groups"};
+  }
+  /** a Unix command to set permission */
+  public static final String SET_PERMISSION_COMMAND = "chmod";
+  /** a Unix command to set owner */
+  public static final String SET_OWNER_COMMAND = "chown";
+  /** Return a Unix command to get permission information. */
+  public static String[] getGET_PERMISSION_COMMAND() {
+    return new String[]{"ls", "-ld"};
+  }
+
+  private long    interval;   // refresh interval in msec
+  private long    lastTime;   // last time the command was performed
+  private Map<String, String> environment; // env for the command execution
+  private File dir;
+  private Process process; // sub process used to execute the command
+  private int exitCode;
+  
+  public Shell() {
+    this(0L);
+  }
+  
+  /**
+   * @param Interval the minimum duration to wait before re-executing the 
+   * command.
+   */
+  public Shell( long interval ) {
+    this.interval = interval;
+    this.lastTime = (interval<0) ? 0 : -interval;
+  }
+  
+  /** set the environment for the command 
+   * @param env Mapping of environment variables
+   */
+  protected void setEnvironment(Map<String, String> env) {
+    this.environment = env;
+  }
+
+  /** set the working directory 
+   * @param dir The directory where the command would be executed
+   */
+  protected void setWorkingDirectory(File dir) {
+    this.dir = dir;
+  }
+
+  /** check to see if a command needs to be executed and execute if needed */
+  protected void run() throws IOException {
+    if (lastTime + interval > System.currentTimeMillis())
+      return;
+    exitCode = 0; // reset for next run
+    runCommand();
+  }
+
+  /** Run a command */
+  private void runCommand() throws IOException { 
+    ProcessBuilder builder = new ProcessBuilder(getExecString());
+    boolean completed = false;
+    
+    if (environment != null) {
+      builder.environment().putAll(this.environment);
+    }
+    if (dir != null) {
+      builder.directory(this.dir);
+    }
+    
+    process = builder.start();
+    final BufferedReader errReader = 
+            new BufferedReader(new InputStreamReader(process
+                                                     .getErrorStream()));
+    BufferedReader inReader = 
+            new BufferedReader(new InputStreamReader(process
+                                                     .getInputStream()));
+    final StringBuffer errMsg = new StringBuffer();
+    
+    // read error and input streams as this would free up the buffers
+    // free the error stream buffer
+    Thread errThread = new Thread() {
+      @Override
+      public void run() {
+        try {
+          String line = errReader.readLine();
+          while((line != null) && !isInterrupted()) {
+            errMsg.append(line);
+            errMsg.append(System.getProperty("line.seperator"));
+            line = errReader.readLine();
+          }
+        } catch(IOException ioe) {
+          LOG.warn("Error reading the error stream", ioe);
+        }
+      }
+    };
+    try {
+      errThread.start();
+    } catch (IllegalStateException ise) { }
+    try {
+      parseExecResult(inReader); // parse the output
+      // clear the input stream buffer
+      String line = inReader.readLine();
+      while(line != null) { 
+        line = inReader.readLine();
+      }
+      // wait for the process to finish and check the exit code
+      exitCode = process.waitFor();
+      if (exitCode != 0) {
+        if (errMsg.length() == 0) {
+          errMsg.append("Command exit with status code " + exitCode);
+        }
+        throw new IOException(errMsg.toString());
+      }
+      completed = true;
+    } catch (InterruptedException ie) {
+      throw new IOException(ie.toString());
+    } finally {
+      // close the input stream
+      try {
+        inReader.close();
+      } catch (IOException ioe) {
+        LOG.warn("Error while closing the input stream", ioe);
+      }
+      if (completed) {
+        try {
+          // make sure that the error thread exits
+          errThread.join();
+        } catch (InterruptedException ie) {
+          LOG.warn("Interrupted while reading the error stream", ie);
+        }
+      } else {
+        errThread.interrupt();
+      }
+      try {
+        errReader.close();
+      } catch (IOException ioe) {
+        LOG.warn("Error while closing the error stream", ioe);
+      }
+      process.destroy();
+      lastTime = System.currentTimeMillis();
+    }
+  }
+
+  /** return an array containing the command name & its parameters */ 
+  protected abstract String[] getExecString();
+  
+  /** Parse the execution result */
+  protected abstract void parseExecResult(BufferedReader lines)
+  throws IOException;
+
+  /** get the current sub-process executing the given command 
+   * @return process executing the command
+   */
+  public Process getProcess() {
+    return process;
+  }
+
+  /** get the exit code 
+   * @return the exit code of the process
+   */
+  public int getExitCode() {
+    return exitCode;
+  }
+
+  /**
+   * A simple shell command executor.
+   * 
+   * <code>ShellCommandExecutor</code>should be used in cases where the output

+   * of the command needs no explicit parsing and where the command, working 
+   * directory and the environment remains unchanged. The output of the command 
+   * is stored as-is and is expected to be small.
+   */
+  public static class ShellCommandExecutor extends Shell {
+    
+    private String[] command;
+    private StringBuffer output;
+    
+    public ShellCommandExecutor(String[] execString) {
+      command = execString.clone();
+    }
+
+    public ShellCommandExecutor(String[] execString, File dir) {
+      this(execString);
+      this.setWorkingDirectory(dir);
+    }
+
+    public ShellCommandExecutor(String[] execString, File dir, 
+                                 Map<String, String> env) {
+      this(execString, dir);
+      this.setEnvironment(env);
+    }
+    
+    /** Execute the shell command. */
+    public void execute() throws IOException {
+      this.run();    
+    }
+
+    protected String[] getExecString() {
+      return command;
+    }
+
+    protected void parseExecResult(BufferedReader lines) throws IOException {
+      output = new StringBuffer();
+      char[] buf = new char[512];
+      int nRead;
+      while ( (nRead = lines.read(buf, 0, buf.length)) > 0 ) {
+        output.append(buf, 0, nRead);
+      }
+    }
+    
+    /** Get the output of the shell command.*/
+    public String getOutput() {
+      return (output == null) ? "" : output.toString();
+    }
+  }
+  
+  /** 
+   * Static method to execute a shell command. 
+   * Covers most of the simple cases without requiring the user to implement  
+   * the <code>Shell</code> interface.
+   * @param cmd shell command to execute.
+   * @return the output of the executed command.
+   */
+  public static String execCommand(String ... cmd) throws IOException {
+    ShellCommandExecutor exec = new ShellCommandExecutor(cmd);
+    exec.execute();
+    return exec.getOutput();
+  }
+}

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ShellUtil.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ShellUtil.java?rev=608050&r1=608049&r2=608050&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ShellUtil.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ShellUtil.java Wed Jan  2 01:19:11
2008
@@ -5,28 +5,24 @@
 import java.io.File;
 import java.io.IOException;
 
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+
 /**
  *  Class to execute a shell.
- *
+ *  @deprecated Use {@link ShellCommandExecutor} instead.
  */
-
 public class ShellUtil {
-    private List<String> exec_cmd;
-    private File working_dir;
-    private Process process;
-    private int exit_code;
-    private Map<String, String> environment;
-
+  
+  ShellCommandExecutor shexec; // shell to execute a command
+  
     /**
      * @param args list containing command and command line arguments
      * @param dir Current working directory
-     * @throws IOException 
-     * @throws InterruptedException
-	 */
+     * @param env Environment for the command
+     */
     public ShellUtil (List<String> args, File dir, Map<String, String> env) {
-      exec_cmd = args;
-      working_dir = dir;
-      environment  = env;
+      shexec = new ShellCommandExecutor(args.toArray(new String[0]), dir, 
+                                         env);
     }
 	
     /**
@@ -34,28 +30,21 @@
      * @throws IOException
      * @throws InterruptedException
      */
-    public void execute() throws IOException, InterruptedException {
-      // start the process and wait for it to execute 
-      ProcessBuilder builder = new ProcessBuilder(exec_cmd);
-      builder.directory(working_dir);
-      if (environment != null) {
-        builder.environment().putAll(environment);
-      }
-      process = builder.start();
-      exit_code = process.waitFor();
-      
+    public void execute() throws IOException {
+      // start the process and wait for it to execute
+      shexec.execute();
     }
     /**
      * @return process
      */
     public Process getProcess() {
-      return process;
+      return shexec.getProcess();
     }
 
     /**
      * @return exit-code of the process
      */
     public int getExitCode() {
-      return exit_code;
+      return shexec.getExitCode();
    }
 }

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java?rev=608050&r1=608049&r2=608050&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java Wed Jan
 2 01:19:11 2008
@@ -27,7 +27,7 @@
 import java.util.zip.CRC32;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.ShellCommand;
+import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.fs.FSInputStream;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.io.UTF8;
@@ -72,7 +72,7 @@
                  FileUtil.makeShellPath(dataDir) + "' ; tar -xf -)";
     LOG.info("Unpacking the tar file. Cmd : " + cmd);
     String[] shellCmd = { "bash", "-c", cmd };
-    ShellCommand.execCommand(shellCmd);
+    Shell.execCommand(shellCmd);
     
     //Now read the reference info
     

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalDirAllocator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalDirAllocator.java?rev=608050&r1=608049&r2=608050&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalDirAllocator.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalDirAllocator.java Wed Jan 
2 01:19:11 2008
@@ -21,6 +21,7 @@
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Shell;
 
 import junit.framework.TestCase;
 
@@ -90,7 +91,7 @@
       validateTempDirCreation(1);
       validateTempDirCreation(1);
     } finally {
-      ShellCommand.execCommand(new String[]{"chmod", "u+w", BUFFER_DIR_ROOT});
+      Shell.execCommand(new String[]{"chmod", "u+w", BUFFER_DIR_ROOT});
       rmBufferDirs();
     }
   }
@@ -108,7 +109,7 @@
       validateTempDirCreation(2);
       validateTempDirCreation(2);
     } finally {
-      ShellCommand.execCommand(new String[]{"chmod", "u+w", BUFFER_DIR_ROOT});
+      Shell.execCommand(new String[]{"chmod", "u+w", BUFFER_DIR_ROOT});
       rmBufferDirs();
     }
   }

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystemPermission.java?rev=608050&r1=608049&r2=608050&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystemPermission.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystemPermission.java Wed
Jan  2 01:19:11 2008
@@ -20,6 +20,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.*;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Shell;
 
 import java.io.*;
 import java.util.*;
@@ -138,7 +139,7 @@
 
   static List<String> getGroups() throws IOException {
     List<String> a = new ArrayList<String>();
-    String s = ShellCommand.execCommand(ShellCommand.getGROUPS_COMMAND());
+    String s = Shell.execCommand(Shell.getGROUPS_COMMAND());
     for(StringTokenizer t = new StringTokenizer(s); t.hasMoreTokens(); ) {
       a.add(t.nextToken());
     }



Mime
View raw message