hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1077555 - in /hadoop/common/branches/branch-0.20-security-patches/src: mapred/org/apache/hadoop/mapred/JobHistory.java mapred/org/apache/hadoop/mapred/JobTracker.java webapps/job/jobhistory.jsp
Date Fri, 04 Mar 2011 04:28:41 GMT
Author: omalley
Date: Fri Mar  4 04:28:40 2011
New Revision: 1077555

URL: http://svn.apache.org/viewvc?rev=1077555&view=rev
Log:
commit fac50cf14ffe968281ba41fecdf4d946d3f942fa
Author: Arun C Murthy <acmurthy@apache.org>
Date:   Mon Jul 19 10:51:07 2010 -0700

    MAPREDUCE-323. Fixed underscores in user and job names, also fixed some debug code. Contributed
by Dick King.

Modified:
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
    hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobhistory.jsp

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=1077555&r1=1077554&r2=1077555&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
Fri Mar  4 04:28:40 2011
@@ -98,6 +98,8 @@ public class JobHistory {
   static final String DONE_DIRECTORY_FORMAT_DIRNAME
     = "version-" + DONE_DIRECTORY_FORMAT_VERSION;
 
+  static final String UNDERSCORE_ESCAPE = "%5F";
+
   public static final Log LOG = LogFactory.getLog(JobHistory.class);
   private static final char DELIMITER = ' ';
   static final char LINE_DELIMITER_CHAR = '.';
@@ -135,7 +137,7 @@ public class JobHistory {
   static final String CONF_FILE_NAME_SUFFIX = "_conf.xml";
 
   // XXXXX debug mode -- set this to false for production
-  private static final boolean DEBUG_MODE = true;
+  private static final boolean DEBUG_MODE = false;
 
   private static final int SERIAL_NUMBER_DIRECTORY_DIGITS = 6;
   private static final int SERIAL_NUMBER_LOW_DIGITS = DEBUG_MODE ? 1 : 3;
@@ -334,7 +336,7 @@ public class JobHistory {
 
   private static String serialNumberDirectoryComponent(JobID id) {
     return String.format(SERIAL_NUMBER_FORMAT,
-                         new Integer(jobSerialNumber(id)))
+                         Integer.valueOf(jobSerialNumber(id)))
               .substring(0, SERIAL_NUMBER_DIRECTORY_DIGITS);
   }
 
@@ -533,10 +535,19 @@ public class JobHistory {
     //permission
     if (!DONEDIR_FS.exists(DONE)) {
       LOG.info("Creating DONE folder at "+ DONE);
-      if (! DONEDIR_FS.mkdirs(DONE, 
+      if (!DONEDIR_FS.mkdirs(DONE, 
           new FsPermission(HISTORY_DIR_PERMISSION))) {
         throw new IOException("Mkdirs failed to create " + DONE.toString());
       }
+
+      Path versionSubdir = new Path(DONE, DONE_DIRECTORY_FORMAT_DIRNAME);
+
+      if (!DONEDIR_FS.exists(versionSubdir)) {
+        if (!DONEDIR_FS.mkdirs(versionSubdir,
+                               new FsPermission(HISTORY_DIR_PERMISSION))) {
+          throw new IOException("Mkdirs failed to create " + versionSubdir);
+        }
+      }
     }
     fileManager.start();
   }
@@ -981,6 +992,44 @@ public class JobHistory {
     String[] jobDetails = logFileName.split("_");
     return jobDetails[4];
   }
+
+
+  // This code will be inefficient if the subject contains dozens of underscores
+  static String escapeUnderscores(String escapee) {
+    return replaceStringInstances(escapee, "_", UNDERSCORE_ESCAPE);
+  }
+
+  static String nonOccursString(String logFileName) {
+    int adHocIndex = 0;
+
+    String unfoundString = "q" + adHocIndex;
+
+    while (logFileName.contains(unfoundString)) {
+      unfoundString = "q" + ++adHocIndex;
+    }
+
+    return unfoundString + "q";
+  }
+
+  // I tolerate this code because I expect a low number of
+  // occurrences in a relatively short string
+  static String replaceStringInstances
+      (String logFileName, String old, String replacement) {
+    int index = logFileName.indexOf(old);
+
+    while (index > 0) {
+      logFileName = (logFileName.substring(0, index)
+                     + replacement
+                     + replaceStringInstances
+                         (logFileName.substring(index + old.length()),
+                          old, replacement));
+
+      index = logFileName.indexOf(old);
+    }
+
+    return logFileName;
+  }      
+
   
   /**
    * Helper class for logging or reading back events related to job start, finish or failure.

@@ -1079,6 +1128,15 @@ public class JobHistory {
      */
     public static String encodeJobHistoryFileName(String logFileName)
     throws IOException {
+      String replacementUnderscoreEscape = null;
+
+      if (logFileName.contains(UNDERSCORE_ESCAPE)) {
+        replacementUnderscoreEscape = nonOccursString(logFileName);
+
+        logFileName = replaceStringInstances
+          (logFileName, UNDERSCORE_ESCAPE, replacementUnderscoreEscape);
+      }
+
       String encodedFileName = null;
       try {
         encodedFileName = URLEncoder.encode(logFileName, "UTF-8");
@@ -1088,6 +1146,12 @@ public class JobHistory {
         ioe.setStackTrace(uee.getStackTrace());
         throw ioe;
       }
+      
+      if (replacementUnderscoreEscape != null) {
+        encodedFileName = replaceStringInstances
+          (encodedFileName, replacementUnderscoreEscape, UNDERSCORE_ESCAPE);
+      }
+
       return encodedFileName;
     }
     
@@ -1173,8 +1237,8 @@ public class JobHistory {
       return
         id.toString() + "_"
         + submitTime + "_"
-        + getUserName(jobConf) + "_" 
-        + trimJobName(getJobName(jobConf));
+        + escapeUnderscores(getUserName(jobConf)) + "_" 
+        + escapeUnderscores(trimJobName(getJobName(jobConf)));
     }
     
     /**
@@ -1226,9 +1290,13 @@ public class JobHistory {
       }
 
       // Make the pattern matching the job's history file
-      final Pattern historyFilePattern = 
-        Pattern.compile(id.toString() + "_" + DIGITS + "_" + user + "_" 
-                        + escapeRegexChars(jobName) + "+");
+
+      final String regexp
+        = id.toString() + "_" + DIGITS + "_" + user + "_"
+             + escapeRegexChars(jobName) + "+";
+      
+      final Pattern historyFilePattern = Pattern.compile(regexp);
+
       // a path filter that matches 4 parts of the filenames namely
       //  - jt-hostname
       //  - job-id
@@ -1236,14 +1304,16 @@ public class JobHistory {
       //  - jobname
       PathFilter filter = new PathFilter() {
         public boolean accept(Path path) {
-          String fileName = path.getName();
+          String unescapedFileName = path.getName();
+          String fileName = null;
           try {
-            fileName = decodeJobHistoryFileName(fileName);
+            fileName = decodeJobHistoryFileName(unescapedFileName);
           } catch (IOException ioe) {
             LOG.info("Error while decoding history file " + fileName + "."
                      + " Ignoring file.", ioe);
             return false;
           }
+
           return historyFilePattern.matcher(fileName).find();
         }
       };

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=1077555&r1=1077554&r2=1077555&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
Fri Mar  4 04:28:40 2011
@@ -2257,11 +2257,11 @@ public class JobTracker implements MRCon
 
         infoServer.setAttribute
           ("serialNumberDirectoryDigits",
-           new Integer(JobHistory.serialNumberDirectoryDigits()));
+           Integer.valueOf(JobHistory.serialNumberDirectoryDigits()));
 
         infoServer.setAttribute
           ("serialNumberTotalDigits",
-           new Integer(JobHistory.serialNumberTotalDigits()));
+           Integer.valueOf(JobHistory.serialNumberTotalDigits()));
         
         return new Path(historyLogDir).getFileSystem(conf);
       }

Modified: hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobhistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobhistory.jsp?rev=1077555&r1=1077554&r2=1077555&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobhistory.jsp (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobhistory.jsp Fri
Mar  4 04:28:40 2011
@@ -64,13 +64,11 @@ window.location.href = url;
 
     final int FILENAME_JOBNAME_PART = FILENAME_JOBID_END + 2;
 
-    // XXXXXXXX debug code -- should start with 20
-    final int[] SCAN_SIZES = { 3, 5, 20, 50, 200 };
+    final int[] SCAN_SIZES = { 20, 50, 200 };
 
     final int FILES_PER_SCAN = 1000;
 
-    // XXXXX debug -- this should be 100.
-    final int DEFAULT_PAGE_SIZE = 10;
+    final int DEFAULT_PAGE_SIZE = 100;
 
     final String DEFAULT_DATE_GLOB_COMPONENT = "*/*/*";
 
@@ -86,47 +84,54 @@ window.location.href = url;
 
     final String parts[] = dateSplit[0].split(":");
 
-    final String user = (parts.length >= 1)
-                        ? parts[0].toLowerCase()
-                        : "";
+    final String rawUser = (parts.length >= 1)
+                            ? parts[0].toLowerCase()
+                            : "";
+
+    final String userInFname
+      = escapeUnderscores(JobHistory.JobInfo.encodeJobHistoryFileName(
+            HtmlQuoting.unquoteHtmlChars(rawUser))).toLowerCase();
 
     final int currentScanSizeIndex
       = (request.getParameter("scansize") == null)
            ? 0 : Integer.parseInt(request.getParameter("scansize"));
 
-    // DEBUG we temporarily allow a "date" with a leading digit of 4 or 5,
-    //  and a "month" with a leading digit of 2, because for testing we will 
-    //  use hours resp. minutes for months resp. days.
     final String SEARCH_PARSE_REGEX
-      = "([0-2]?[0-9])/([0-5]?[0-9])/((?:2[0-9])[0-9][0-9])";
+      = "([0-1]?[0-9])/([0-3]?[0-9])/((?:2[0-9])[0-9][0-9])";
 
     final Pattern dateSearchParse = Pattern.compile(SEARCH_PARSE_REGEX);
 
-    final String jobname = (parts.length >= 2)
-                           ? parts[1].toLowerCase()
-                           : "";
+    final String rawJobname = (parts.length >= 2)
+                               ? parts[1].toLowerCase()
+                               : "";
+
+    final String jobnameKeywordInFname
+      = escapeUnderscores(JobHistory.JobInfo.encodeJobHistoryFileName(
+            HtmlQuoting.unquoteHtmlChars(rawJobname))).toLowerCase();
+
     PathFilter jobLogFileFilter = new PathFilter() {
-      // unquote params before encoding for search
-      final String uqUser = JobHistory.JobInfo.encodeJobHistoryFileName(
-            HtmlQuoting.unquoteHtmlChars(user));
-      final String uqJobname = JobHistory.JobInfo.encodeJobHistoryFileName(
-            HtmlQuoting.unquoteHtmlChars(jobname));
       private boolean matchUser(String fileName) {
         // return true if 
         //  - user is not specified
         //  - user matches
-        return "".equals(uqUser) || uqUser.equals(fileName.split("_")[FILENAME_USER_PART]);
+        return "".equals(userInFname)
+           || userInFname.equals(fileName.split("_")[FILENAME_USER_PART]
+                .toLowerCase());
       }
 
       private boolean matchJobName(String fileName) {
         // return true if 
         //  - jobname is not specified
         //  - jobname contains the keyword
-        return "".equals(uqJobname) || fileName.split("_")[FILENAME_JOBNAME_PART].toLowerCase().contains(uqJobname);
+        return "".equals(jobnameKeywordInFname) 
+                 || fileName.split("_")[FILENAME_JOBNAME_PART].toLowerCase()
+                       .contains(jobnameKeywordInFname);
       }
 
       public boolean accept(Path path) {
-        return !(path.getName().endsWith(".xml")) && matchUser(path.getName()) &&
matchJobName(path.getName());
+        String name = path.getName();
+
+        return !(name.endsWith(".xml")) && matchUser(name) && matchJobName(name);
       }
     };
     
@@ -290,8 +295,8 @@ window.location.href = url;
                   + "</h2>");
     }
 
-    out.println("<!--  user : " + user +
-        ", jobname : " + jobname + "-->");
+    out.println("<!--  user : " + rawUser +
+        ", jobname : " + rawJobname + "-->");
     if (null == jobFiles || jobFiles.length == 0)  {
       out.println("No files found!"); 
       return ; 
@@ -364,13 +369,13 @@ window.location.href = url;
                 + (sizeIsExact
                    ? ""
                    : ", <b>" + numHistoryFiles + "</b> gotten"));
-    if (!"".equals(user)) {
+    if (!"".equals(rawUser)) {
       // show the user if present
-      out.println(" for user <b>" + user + "</b>");
+      out.println(" for user <b>" + rawUser + "</b>");
     }
-    if (!"".equals(jobname)) {
+    if (!"".equals(rawJobname)) {
       out.println(" with jobname having the keyword <b>" +
-          jobname + "</b> in it.");
+          rawJobname + "</b> in it.");
       // show the jobname keyword if present
     }
     if (!DEFAULT_DATE_GLOB_COMPONENT.equals(dateComponent)) {
@@ -430,9 +435,14 @@ window.location.href = url;
     Set<String> displayedJobs = new HashSet<String>();
     for (int i = start - 1; i < start + length - 1; ++i) {
       Path jobFile = jobFiles[i];
+
+      String fname = jobFile.getName();
+      String marker = JobHistory.nonOccursString(fname);
+      String reescapedFname = JobHistory.replaceStringInstances(fname,
+                  JobHistory.UNDERSCORE_ESCAPE, marker);
       
       String decodedJobFileName = 
-          JobHistory.JobInfo.decodeJobHistoryFileName(jobFile.getName());
+          JobHistory.JobInfo.decodeJobHistoryFileName(reescapedFname);
 
       String[] jobDetails = decodedJobFileName.split("_");
       String trackerStartTime = jobDetails[1];
@@ -440,8 +450,10 @@ window.location.href = url;
                       + "_" + jobDetails[JOB_ID_START + 1]
                       + "_" + jobDetails[JOB_ID_START + 2]);
       String submitTimestamp = jobDetails[FILENAME_SUBMIT_TIMESTAMP_PART];
-      String userName = jobDetails[FILENAME_USER_PART];
-      String jobName = jobDetails[FILENAME_JOBNAME_PART];
+      String userName = JobHistory.replaceStringInstances(jobDetails[FILENAME_USER_PART],
+                  marker, JobHistory.UNDERSCORE_ESCAPE);
+      String jobName = JobHistory.replaceStringInstances(jobDetails[FILENAME_JOBNAME_PART],
+                  marker, JobHistory.UNDERSCORE_ESCAPE);
       
       // Check if the job is already displayed. There can be multiple job 
       // history files for jobs that have restarted
@@ -452,8 +464,11 @@ window.location.href = url;
       }
       
       // Encode the logfile name again to cancel the decoding done by the browser
-      String encodedJobFileName = 
+      String preEncodedJobFileName = 
           JobHistory.JobInfo.encodeJobHistoryFileName(jobFile.getName());
+
+      String encodedJobFileName = 
+          JobHistory.replaceStringInstances(preEncodedJobFileName, "%5F", "%255F");
 %>
 <center>
 <%	
@@ -478,11 +493,37 @@ window.location.href = url;
       out.print("<td>" + new Date(Long.parseLong(timestamp)) + "</td>"); 
       out.print("<td>" + "<a href=\"jobdetailshistory.jsp?logFile=" 
           + logFile.toString() + "\">" + jobId + "</a></td>");
-      out.print("<td>" + HtmlQuoting.quoteHtmlChars(jobName) + "</td>"); 
-      out.print("<td>" + HtmlQuoting.quoteHtmlChars(user) + "</td>"); 
+      out.print("<td>"
+                + HtmlQuoting.quoteHtmlChars(unescapeUnderscores(jobName))
+                + "</td>"); 
+      out.print("<td>"
+                + HtmlQuoting.quoteHtmlChars(unescapeUnderscores(user))
+                + "</td>"); 
       out.print("</tr>");
     }
 
+    private String escapeUnderscores(String rawString) {
+      return convertStrings(rawString, "_", "%5F");
+    }
+
+    private String unescapeUnderscores(String rawString) {
+      return convertStrings(rawString, "%5F", "_");
+    }
+
+    // inefficient if there are a lot of underscores
+    private String convertStrings(String escapedString, String from, String to) {
+      int firstEscape = escapedString.indexOf(from);
+
+      if (firstEscape < 0) {
+        return escapedString;
+      }
+
+      return escapedString.substring(0, firstEscape)
+            + to
+            + unescapeUnderscores(escapedString.substring
+                                    (firstEscape + from.length()));
+    }
+
     private void printNavigationTool(int pageno, int size, int max,
                                      String searchPlusScan, JspWriter out)
          throws IOException {



Mime
View raw message