hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject svn commit: r1507378 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/ hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java...
Date Fri, 26 Jul 2013 17:32:00 GMT
Author: jlowe
Date: Fri Jul 26 17:32:00 2013
New Revision: 1507378

URL: http://svn.apache.org/r1507378
Log:
svn merge -c 1507135 FIXES: MAPREDUCE-5386. Ability to refresh history server job retention
and job cleaner settings. Contributed by Ashwin Shankar

Added:
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistory.java
      - copied unchanged from r1507135, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistory.java
Modified:
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocol/HSAdminRefreshProtocol.java
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolClientSideTranslatorPB.java
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolServerSideTranslatorPB.java
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/proto/HSAdminRefreshProtocol.proto
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1507378&r1=1507377&r2=1507378&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Fri Jul 26 17:32:00
2013
@@ -12,6 +12,9 @@ Release 2.3.0 - UNRELEASED
     MAPREDUCE-5356. Ability to refresh aggregated log retention period and
     check interval (Ashwin Shankar via jlowe)
 
+    MAPREDUCE-5386. Ability to refresh history server job retention and job
+    cleaner settings (Ashwin Shankar via jlowe)
+
   IMPROVEMENTS
 
   OPTIMIZATIONS

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java?rev=1507378&r1=1507377&r2=1507378&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
Fri Jul 26 17:32:00 2013
@@ -52,6 +52,7 @@ import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.mapred.JobACLsManager;
 import org.apache.hadoop.mapreduce.jobhistory.JobSummary;
@@ -61,6 +62,7 @@ import org.apache.hadoop.mapreduce.v2.jo
 import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 
@@ -416,7 +418,7 @@ public class HistoryFileManager extends 
       return historyFile;
     }
     
-    private synchronized void delete() throws IOException {
+    protected synchronized void delete() throws IOException {
       if (LOG.isDebugEnabled()) {
         LOG.debug("deleting " + historyFile + " and " + confFile);
       }
@@ -524,10 +526,7 @@ public class HistoryFileManager extends 
     maxHistoryAge = conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS,
         JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);
     
-    jobListCache = new JobListCache(conf.getInt(
-        JHAdminConfig.MR_HISTORY_JOBLIST_CACHE_SIZE,
-        JHAdminConfig.DEFAULT_MR_HISTORY_JOBLIST_CACHE_SIZE),
-        maxHistoryAge);
+    jobListCache = createJobListCache();
 
     serialNumberIndex = new SerialNumberIndex(conf.getInt(
         JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
@@ -544,6 +543,12 @@ public class HistoryFileManager extends 
     super.serviceInit(conf);
   }
 
+  protected JobListCache createJobListCache() {
+    return new JobListCache(conf.getInt(
+        JHAdminConfig.MR_HISTORY_JOBLIST_CACHE_SIZE,
+        JHAdminConfig.DEFAULT_MR_HISTORY_JOBLIST_CACHE_SIZE), maxHistoryAge);
+  }
+
   private void mkdir(FileContext fc, Path path, FsPermission fsp)
       throws IOException {
     if (!fc.util().exists(path)) {
@@ -656,18 +661,18 @@ public class HistoryFileManager extends 
     return jhStatusList;
   }
 
-  private static List<FileStatus> scanDirectoryForHistoryFiles(Path path,
+  protected List<FileStatus> scanDirectoryForHistoryFiles(Path path,
       FileContext fc) throws IOException {
     return scanDirectory(path, fc, JobHistoryUtils.getHistoryFileFilter());
   }
-
+  
   /**
    * Finds all history directories with a timestamp component by scanning the
    * filesystem. Used when the JobHistory server is started.
    * 
-   * @return
+   * @return list of history directories
    */
-  private List<FileStatus> findTimestampedDirectories() throws IOException {
+  protected List<FileStatus> findTimestampedDirectories() throws IOException {
     List<FileStatus> fsList = JobHistoryUtils.localGlobber(doneDirFc,
         doneDirPrefixPath, DONE_BEFORE_SERIAL_TAIL);
     return fsList;
@@ -954,7 +959,7 @@ public class HistoryFileManager extends 
         }
       }
       if (!halted) {
-        doneDirFc.delete(doneDirFc.makeQualified(serialDir.getPath()), true);
+        deleteDir(serialDir);
         removeDirectoryFromSerialNumberIndex(serialDir.getPath());
         existingDoneSubdirs.remove(serialDir.getPath());
       } else {
@@ -962,6 +967,13 @@ public class HistoryFileManager extends 
       }
     }
   }
+  
+  protected boolean deleteDir(FileStatus serialDir)
+      throws AccessControlException, FileNotFoundException,
+      UnsupportedFileSystemException, IOException {
+    return doneDirFc.delete(doneDirFc.makeQualified(serialDir.getPath()), true);
+  }
+
   @VisibleForTesting
   protected void setMaxHistoryAge(long newValue){
     maxHistoryAge=newValue;

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java?rev=1507378&r1=1507377&r2=1507378&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
Fri Jul 26 17:32:00 2013
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
@@ -71,7 +72,11 @@ public class JobHistory extends Abstract
 
   private HistoryStorage storage = null;
   private HistoryFileManager hsManager = null;
-
+  ScheduledFuture<?> futureHistoryCleaner = null;
+  
+  //History job cleaner interval
+  private long cleanerInterval;
+  
   @Override
   protected void serviceInit(Configuration conf) throws Exception {
     LOG.info("JobHistory Init");
@@ -84,7 +89,7 @@ public class JobHistory extends Abstract
         JHAdminConfig.MR_HISTORY_MOVE_INTERVAL_MS,
         JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_INTERVAL_MS);
 
-    hsManager = new HistoryFileManager();
+    hsManager = createHistoryFileManager();
     hsManager.init(conf);
     try {
       hsManager.initExisting();
@@ -103,6 +108,10 @@ public class JobHistory extends Abstract
     super.serviceInit(conf);
   }
 
+  protected HistoryFileManager createHistoryFileManager() {
+    return new HistoryFileManager();
+  }
+
   @Override
   protected void serviceStart() throws Exception {
     hsManager.start();
@@ -118,19 +127,14 @@ public class JobHistory extends Abstract
         moveThreadInterval, moveThreadInterval, TimeUnit.MILLISECONDS);
 
     // Start historyCleaner
-    boolean startCleanerService = conf.getBoolean(
-        JHAdminConfig.MR_HISTORY_CLEANER_ENABLE, true);
-    if (startCleanerService) {
-      long runInterval = conf.getLong(
-          JHAdminConfig.MR_HISTORY_CLEANER_INTERVAL_MS,
-          JHAdminConfig.DEFAULT_MR_HISTORY_CLEANER_INTERVAL_MS);
-      scheduledExecutor
-          .scheduleAtFixedRate(new HistoryCleaner(),
-              30 * 1000l, runInterval, TimeUnit.MILLISECONDS);
-    }
+    scheduleHistoryCleaner();
     super.serviceStart();
   }
 
+  protected int getInitDelaySecs() {
+    return 30;
+  }
+  
   @Override
   protected void serviceStop() throws Exception {
     LOG.info("Stopping JobHistory");
@@ -256,6 +260,43 @@ public class JobHistory extends Abstract
         fBegin, fEnd, jobState);
   }
 
+  public void refreshJobRetentionSettings() {
+    if (getServiceState() == STATE.STARTED) {
+      conf = createConf();
+      long maxHistoryAge = conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS,
+          JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);
+      hsManager.setMaxHistoryAge(maxHistoryAge);
+      if (futureHistoryCleaner != null) {
+        futureHistoryCleaner.cancel(false);
+      }
+      futureHistoryCleaner = null;
+      scheduleHistoryCleaner();
+    } else {
+      LOG.warn("Failed to execute refreshJobRetentionSettings : Job History service is not
started");
+    }
+  }
+
+  private void scheduleHistoryCleaner() {
+    boolean startCleanerService = conf.getBoolean(
+        JHAdminConfig.MR_HISTORY_CLEANER_ENABLE, true);
+    if (startCleanerService) {
+      cleanerInterval = conf.getLong(
+          JHAdminConfig.MR_HISTORY_CLEANER_INTERVAL_MS,
+          JHAdminConfig.DEFAULT_MR_HISTORY_CLEANER_INTERVAL_MS);
+
+      futureHistoryCleaner = scheduledExecutor.scheduleAtFixedRate(
+          new HistoryCleaner(), getInitDelaySecs() * 1000l, cleanerInterval,
+          TimeUnit.MILLISECONDS);
+    }
+  }
+
+  protected Configuration createConf() {
+    return new Configuration();
+  }
+  
+  public long getCleanerInterval() {
+    return cleanerInterval;
+  }
   // TODO AppContext - Not Required
   private ApplicationAttemptId appAttemptID;
 

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java?rev=1507378&r1=1507377&r2=1507378&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
Fri Jul 26 17:32:00 2013
@@ -83,7 +83,7 @@ public class JobHistoryServer extends Co
     clientService = new HistoryClientService(historyContext, 
         this.jhsDTSecretManager);
     aggLogDelService = new AggregatedLogDeletionService();
-    hsAdminServer = new HSAdminServer(aggLogDelService);
+    hsAdminServer = new HSAdminServer(aggLogDelService, jobHistoryService);
     addService(jobHistoryService);
     addService(clientService);
     addService(aggLogDelService);

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java?rev=1507378&r1=1507377&r2=1507378&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java
Fri Jul 26 17:32:00 2013
@@ -60,8 +60,12 @@ public class HSAdmin extends Configured 
           .println("Usage: mapred hsadmin [-refreshSuperUserGroupsConfiguration]");
     } else if ("-refreshAdminAcls".equals(cmd)) {
       System.err.println("Usage: mapred hsadmin [-refreshAdminAcls]");
+    } else if ("-refreshJobRetentionSettings".equals(cmd)) {
+      System.err
+          .println("Usage: mapred hsadmin [-refreshJobRetentionSettings]");
     } else if ("-refreshLogRetentionSettings".equals(cmd)) {
-      System.err.println("Usage: mapred hsadmin [-refreshLogRetentionSettings]");
+      System.err
+          .println("Usage: mapred hsadmin [-refreshLogRetentionSettings]");
     } else if ("-getGroups".equals(cmd)) {
       System.err.println("Usage: mapred hsadmin" + " [-getGroups [username]]");
     } else {
@@ -69,6 +73,7 @@ public class HSAdmin extends Configured 
       System.err.println("           [-refreshUserToGroupsMappings]");
       System.err.println("           [-refreshSuperUserGroupsConfiguration]");
       System.err.println("           [-refreshAdminAcls]");
+      System.err.println("           [-refreshJobRetentionSettings]");
       System.err.println("           [-refreshLogRetentionSettings]");
       System.err.println("           [-getGroups [username]]");
       System.err.println("           [-help [cmd]]");
@@ -84,6 +89,8 @@ public class HSAdmin extends Configured 
         + " [-refreshUserToGroupsMappings]"
         + " [-refreshSuperUserGroupsConfiguration]"
         + " [-refreshAdminAcls]"
+        + " [-refreshLogRetentionSettings]"
+        + " [-refreshJobRetentionSettings]"
         + " [-getGroups [username]]" + " [-help [cmd]]\n";
 
     String refreshUserToGroupsMappings = "-refreshUserToGroupsMappings: Refresh user-to-groups
mappings\n";
@@ -92,8 +99,13 @@ public class HSAdmin extends Configured 
 
     String refreshAdminAcls = "-refreshAdminAcls: Refresh acls for administration of Job
history server\n";
 
-    String refreshLogRetentionSettings = "-refreshLogRetentionSettings: Refresh 'log retention
time' and 'log retention check interval' \n";
+    String refreshJobRetentionSettings = "-refreshJobRetentionSettings:" + 
+        "Refresh job history period,job cleaner settings\n";
+
+    String refreshLogRetentionSettings = "-refreshLogRetentionSettings:" + 
+        "Refresh log retention period and log retention check interval\n";
     
+
     String getGroups = "-getGroups [username]: Get the groups which given user belongs to\n";
 
     String help = "-help [cmd]: \tDisplays help for the given command or all commands if
none\n"
@@ -107,6 +119,8 @@ public class HSAdmin extends Configured 
       System.out.println(refreshSuperUserGroupsConfiguration);
     } else if ("refreshAdminAcls".equals(cmd)) {
       System.out.println(refreshAdminAcls);
+    } else if ("refreshJobRetentionSettings".equals(cmd)) {
+      System.out.println(refreshJobRetentionSettings);
     } else if ("refreshLogRetentionSettings".equals(cmd)) {
       System.out.println(refreshLogRetentionSettings);
     } else if ("getGroups".equals(cmd)) {
@@ -116,6 +130,7 @@ public class HSAdmin extends Configured 
       System.out.println(refreshUserToGroupsMappings);
       System.out.println(refreshSuperUserGroupsConfiguration);
       System.out.println(refreshAdminAcls);
+      System.out.println(refreshJobRetentionSettings);
       System.out.println(refreshLogRetentionSettings);
       System.out.println(getGroups);
       System.out.println(help);
@@ -201,11 +216,27 @@ public class HSAdmin extends Configured 
     HSAdminRefreshProtocol refreshProtocol = HSProxies.createProxy(conf,
         address, HSAdminRefreshProtocol.class,
         UserGroupInformation.getCurrentUser());
-    // Refresh the user-to-groups mappings
+
     refreshProtocol.refreshAdminAcls();
     return 0;
   }
 
+  private int refreshJobRetentionSettings() throws IOException {
+    // Refresh job retention settings
+    Configuration conf = getConf();
+    InetSocketAddress address = conf.getSocketAddr(
+        JHAdminConfig.JHS_ADMIN_ADDRESS,
+        JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
+        JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);
+
+    HSAdminRefreshProtocol refreshProtocol = HSProxies.createProxy(conf,
+        address, HSAdminRefreshProtocol.class,
+        UserGroupInformation.getCurrentUser());
+
+    refreshProtocol.refreshJobRetentionSettings();
+    return 0;
+  }
+
   private int refreshLogRetentionSettings() throws IOException {
     // Refresh log retention settings
     Configuration conf = getConf();
@@ -214,14 +245,14 @@ public class HSAdmin extends Configured 
         JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
         JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);
 
-    HSAdminRefreshProtocol refreshProtocol = HSProxies
-        .createProxy(conf, address, HSAdminRefreshProtocol.class,
-            UserGroupInformation.getCurrentUser());
+    HSAdminRefreshProtocol refreshProtocol = HSProxies.createProxy(conf,
+        address, HSAdminRefreshProtocol.class,
+        UserGroupInformation.getCurrentUser());
 
     refreshProtocol.refreshLogRetentionSettings();
     return 0;
   }
-  
+
   @Override
   public int run(String[] args) throws Exception {
     if (args.length < 1) {
@@ -236,6 +267,7 @@ public class HSAdmin extends Configured 
     if ("-refreshUserToGroupsMappings".equals(cmd)
         || "-refreshSuperUserGroupsConfiguration".equals(cmd)
         || "-refreshAdminAcls".equals(cmd)
+        || "-refreshJobRetentionSettings".equals(cmd)
         || "-refreshLogRetentionSettings".equals(cmd)) {
       if (args.length != 1) {
         printUsage(cmd);
@@ -250,6 +282,8 @@ public class HSAdmin extends Configured 
       exitCode = refreshSuperUserGroupsConfiguration();
     } else if ("-refreshAdminAcls".equals(cmd)) {
       exitCode = refreshAdminAcls();
+    } else if ("-refreshJobRetentionSettings".equals(cmd)) {
+      exitCode = refreshJobRetentionSettings();
     } else if ("-refreshLogRetentionSettings".equals(cmd)) {
       exitCode = refreshLogRetentionSettings();
     } else if ("-getGroups".equals(cmd)) {

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocol/HSAdminRefreshProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocol/HSAdminRefreshProtocol.java?rev=1507378&r1=1507377&r2=1507378&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocol/HSAdminRefreshProtocol.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocol/HSAdminRefreshProtocol.java
Fri Jul 26 17:32:00 2013
@@ -39,6 +39,13 @@ public interface HSAdminRefreshProtocol 
    * @throws IOException
    */
   public void refreshAdminAcls() throws IOException;
+  
+  /**
+   * Refresh job retention settings.
+   * 
+   * @throws IOException
+   */
+  public void refreshJobRetentionSettings() throws IOException;
 
   /**
    * Refresh log retention settings.

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolClientSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolClientSideTranslatorPB.java?rev=1507378&r1=1507377&r2=1507378&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolClientSideTranslatorPB.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolClientSideTranslatorPB.java
Fri Jul 26 17:32:00 2013
@@ -27,6 +27,7 @@ import org.apache.hadoop.ipc.ProtocolMet
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RpcClientUtil;
 import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshAdminAclsRequestProto;
+import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshJobRetentionSettingsRequestProto;
 import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshLogRetentionSettingsRequestProto;
 import org.apache.hadoop.mapreduce.v2.hs.protocol.HSAdminRefreshProtocol;
 
@@ -42,12 +43,18 @@ public class HSAdminRefreshProtocolClien
 
   private final HSAdminRefreshProtocolPB rpcProxy;
 
-  private final static RefreshAdminAclsRequestProto VOID_REFRESH_ADMIN_ACLS_REQUEST = RefreshAdminAclsRequestProto
+  private final static RefreshAdminAclsRequestProto 
+    VOID_REFRESH_ADMIN_ACLS_REQUEST = RefreshAdminAclsRequestProto
       .newBuilder().build();
   
-  private final static RefreshLogRetentionSettingsRequestProto VOID_REFRESH_LOG_RETENTION_SETTINGS_REQUEST
= RefreshLogRetentionSettingsRequestProto
-      .newBuilder().build();
+  private final static RefreshJobRetentionSettingsRequestProto 
+    VOID_REFRESH_JOB_RETENTION_SETTINGS_REQUEST = 
+       RefreshJobRetentionSettingsRequestProto.newBuilder().build();
   
+  private final static RefreshLogRetentionSettingsRequestProto 
+    VOID_REFRESH_LOG_RETENTION_SETTINGS_REQUEST = 
+      RefreshLogRetentionSettingsRequestProto.newBuilder().build();
+
   public HSAdminRefreshProtocolClientSideTranslatorPB(
       HSAdminRefreshProtocolPB rpcProxy) {
     this.rpcProxy = rpcProxy;
@@ -69,6 +76,16 @@ public class HSAdminRefreshProtocolClien
   }
 
   @Override
+  public void refreshJobRetentionSettings() throws IOException {
+    try {
+      rpcProxy.refreshJobRetentionSettings(NULL_CONTROLLER,
+          VOID_REFRESH_JOB_RETENTION_SETTINGS_REQUEST);
+    } catch (ServiceException se) {
+      throw ProtobufHelper.getRemoteException(se);
+    }
+  }
+
+  @Override
   public void refreshLogRetentionSettings() throws IOException {
     try {
       rpcProxy.refreshLogRetentionSettings(NULL_CONTROLLER,
@@ -77,7 +94,7 @@ public class HSAdminRefreshProtocolClien
       throw ProtobufHelper.getRemoteException(se);
     }
   }
-  
+
   @Override
   public boolean isMethodSupported(String methodName) throws IOException {
     return RpcClientUtil.isMethodSupported(rpcProxy,

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolServerSideTranslatorPB.java?rev=1507378&r1=1507377&r2=1507378&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolServerSideTranslatorPB.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolServerSideTranslatorPB.java
Fri Jul 26 17:32:00 2013
@@ -23,6 +23,8 @@ import java.io.IOException;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshAdminAclsResponseProto;
 import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshAdminAclsRequestProto;
+import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshJobRetentionSettingsRequestProto;
+import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshJobRetentionSettingsResponseProto;
 import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshLogRetentionSettingsRequestProto;
 import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshLogRetentionSettingsResponseProto;
 import org.apache.hadoop.mapreduce.v2.hs.protocol.HSAdminRefreshProtocol;
@@ -36,10 +38,17 @@ public class HSAdminRefreshProtocolServe
 
   private final HSAdminRefreshProtocol impl;
 
-  private final static RefreshAdminAclsResponseProto VOID_REFRESH_ADMIN_ACLS_RESPONSE = RefreshAdminAclsResponseProto
-      .newBuilder().build();
-  private final static RefreshLogRetentionSettingsResponseProto VOID_REFRESH_LOG_RETENTION_SETTINGS_RESPONSE
= RefreshLogRetentionSettingsResponseProto
+  private final static RefreshAdminAclsResponseProto 
+    VOID_REFRESH_ADMIN_ACLS_RESPONSE = RefreshAdminAclsResponseProto
       .newBuilder().build();
+  
+  private final static RefreshJobRetentionSettingsResponseProto 
+    VOID_REFRESH_JOB_RETENTION_SETTINGS_RESPONSE = 
+      RefreshJobRetentionSettingsResponseProto.newBuilder().build();
+  
+  private final static RefreshLogRetentionSettingsResponseProto 
+    VOID_REFRESH_LOG_RETENTION_SETTINGS_RESPONSE = 
+      RefreshLogRetentionSettingsResponseProto.newBuilder().build();
 
   public HSAdminRefreshProtocolServerSideTranslatorPB(
       HSAdminRefreshProtocol impl) {
@@ -59,8 +68,22 @@ public class HSAdminRefreshProtocolServe
   }
 
   @Override
+  public RefreshJobRetentionSettingsResponseProto refreshJobRetentionSettings(
+      RpcController controller, 
+      RefreshJobRetentionSettingsRequestProto request)
+      throws ServiceException {
+    try {
+      impl.refreshJobRetentionSettings();
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+    return VOID_REFRESH_JOB_RETENTION_SETTINGS_RESPONSE;
+  }
+
+  @Override
   public RefreshLogRetentionSettingsResponseProto refreshLogRetentionSettings(
-      RpcController controller, RefreshLogRetentionSettingsRequestProto request)
+      RpcController controller, 
+      RefreshLogRetentionSettingsRequestProto request)
       throws ServiceException {
     try {
       impl.refreshLogRetentionSettings();

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java?rev=1507378&r1=1507377&r2=1507378&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
Fri Jul 26 17:32:00 2013
@@ -44,6 +44,7 @@ import org.apache.hadoop.tools.protocolP
 import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolServerSideTranslatorPB;
 import org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger;
 import org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger.AuditConstants;
+import org.apache.hadoop.mapreduce.v2.hs.JobHistory;
 import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.HSAdminRefreshProtocolService;
 import org.apache.hadoop.mapreduce.v2.hs.protocol.HSAdminProtocol;
 import org.apache.hadoop.mapreduce.v2.hs.protocolPB.HSAdminRefreshProtocolPB;
@@ -62,10 +63,13 @@ public class HSAdminServer extends Abstr
   protected RPC.Server clientRpcServer;
   protected InetSocketAddress clientRpcAddress;
   private static final String HISTORY_ADMIN_SERVER = "HSAdminServer";
-  
-  public HSAdminServer(AggregatedLogDeletionService aggLogDelService) {
+  private JobHistory jobHistoryService = null;
+
+  public HSAdminServer(AggregatedLogDeletionService aggLogDelService,
+      JobHistory jobHistoryService) {
     super(HSAdminServer.class.getName());
     this.aggLogDelService = aggLogDelService;
+    this.jobHistoryService = jobHistoryService;
   }
 
   @Override
@@ -100,7 +104,8 @@ public class HSAdminServer extends Abstr
         .setPort(clientRpcAddress.getPort()).setVerbose(false).build();
 
     addProtocol(conf, GetUserMappingsProtocolPB.class, getUserMappingService);
-    addProtocol(conf, HSAdminRefreshProtocolPB.class, refreshHSAdminProtocolService);
+    addProtocol(conf, HSAdminRefreshProtocolPB.class,
+        refreshHSAdminProtocolService);
 
     adminAcl = new AccessControlList(conf.get(JHAdminConfig.JHS_ADMIN_ACL,
         JHAdminConfig.DEFAULT_JHS_ADMIN_ACL));
@@ -196,7 +201,7 @@ public class HSAdminServer extends Abstr
     HSAuditLogger.logSuccess(user.getShortUserName(), "refreshAdminAcls",
         HISTORY_ADMIN_SERVER);
   }
- 
+
   @Override
   public void refreshLogRetentionSettings() throws IOException {
     UserGroupInformation user = checkAcls("refreshLogRetentionSettings");
@@ -206,4 +211,14 @@ public class HSAdminServer extends Abstr
     HSAuditLogger.logSuccess(user.getShortUserName(),
         "refreshLogRetentionSettings", "HSAdminServer");
   }
+
+  @Override
+  public void refreshJobRetentionSettings() throws IOException {
+    UserGroupInformation user = checkAcls("refreshJobRetentionSettings");
+
+    jobHistoryService.refreshJobRetentionSettings();
+
+    HSAuditLogger.logSuccess(user.getShortUserName(),
+        "refreshJobRetentionSettings", HISTORY_ADMIN_SERVER);
+  }
 }

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/proto/HSAdminRefreshProtocol.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/proto/HSAdminRefreshProtocol.proto?rev=1507378&r1=1507377&r2=1507378&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/proto/HSAdminRefreshProtocol.proto
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/proto/HSAdminRefreshProtocol.proto
Fri Jul 26 17:32:00 2013
@@ -34,6 +34,18 @@ message RefreshAdminAclsResponseProto {
 }
 
 /**
+ *  refresh job retention settings request.
+ */
+message RefreshJobRetentionSettingsRequestProto {
+}
+
+/**
+ * Response for refresh job retention.
+ */
+message RefreshJobRetentionSettingsResponseProto {
+}
+
+/*
  *  refresh log retention request.
  */
 message RefreshLogRetentionSettingsRequestProto {
@@ -54,6 +66,13 @@ service HSAdminRefreshProtocolService {
    */
   rpc refreshAdminAcls(RefreshAdminAclsRequestProto)
       returns(RefreshAdminAclsResponseProto);
+      
+  /**
+   * Refresh job retention.
+   */
+  rpc refreshJobRetentionSettings(RefreshJobRetentionSettingsRequestProto)
+      returns(RefreshJobRetentionSettingsResponseProto);
+
    /**
    * Refresh log retention
    */

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java?rev=1507378&r1=1507377&r2=1507378&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
Fri Jul 26 17:32:00 2013
@@ -28,6 +28,7 @@ import java.util.List;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.mapreduce.v2.hs.JobHistory;
 import org.apache.hadoop.mapreduce.v2.hs.client.HSAdmin;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
 import org.apache.hadoop.security.GroupMappingServiceProvider;
@@ -49,6 +50,7 @@ public class TestHSAdminServer {
   private HSAdmin hsAdminClient = null;
   Configuration conf = null;
   private static long groupRefreshTimeoutSec = 1;
+  JobHistory jobHistoryService = null;
   AggregatedLogDeletionService alds = null;
 
   public static class MockUnixGroupsMapping implements
@@ -85,9 +87,11 @@ public class TestHSAdminServer {
         GroupMappingServiceProvider.class);
     conf.setLong("hadoop.security.groups.cache.secs", groupRefreshTimeoutSec);
     Groups.getUserToGroupsMappingService(conf);
+    jobHistoryService = mock(JobHistory.class);
     alds = mock(AggregatedLogDeletionService.class);
 
-    hsAdminServer = new HSAdminServer(alds) {
+    hsAdminServer = new HSAdminServer(alds, jobHistoryService) {
+
       @Override
       protected Configuration createConf() {
         return conf;
@@ -236,13 +240,21 @@ public class TestHSAdminServer {
     }
     assertTrue(th instanceof RemoteException);
   }
-  
+
   @Test
   public void testRefreshLogRetentionSettings() throws Exception {
-	String[] args = new String[1];
-	args[0] = "-refreshLogRetentionSettings";
-	hsAdminClient.run(args);
-	verify(alds).refreshLogRetentionSettings();
+    String[] args = new String[1];
+    args[0] = "-refreshLogRetentionSettings";
+    hsAdminClient.run(args);
+    verify(alds).refreshLogRetentionSettings();
+  }
+
+  @Test
+  public void testRefreshJobRetentionSettings() throws Exception {
+    String[] args = new String[1];
+    args[0] = "-refreshJobRetentionSettings";
+    hsAdminClient.run(args);
+    verify(jobHistoryService).refreshJobRetentionSettings();
   }
 
   @After



Mime
View raw message