Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: yufei@apache.org
To: common-commits@hadoop.apache.org
Message-Id: <0beb77f4533e4715816dcb863e591186@git.apache.org>
Subject: hadoop git commit: YARN-7524. Remove unused FairSchedulerEventLog.
 (Contributed by Wilfred Spiegelenburg)
Date: Wed, 22 Nov 2017 22:18:52 +0000 (UTC)
archived-at: Wed, 22 Nov 2017 22:18:56 -0000

Repository: hadoop
Updated Branches:
  refs/heads/trunk 738d1a206 -> 4cc9479da


YARN-7524. Remove unused FairSchedulerEventLog. (Contributed by Wilfred Spiegelenburg)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4cc9479d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4cc9479d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4cc9479d

Branch: refs/heads/trunk
Commit: 4cc9479dae2bfb7d14d29b55d103eea9fa35a586
Parents: 738d1a2
Author: Yufei Gu <yufei@apache.org>
Authored: Wed Nov 22 14:18:36 2017 -0800
Committer: Yufei Gu <yufei@apache.org>
Committed: Wed Nov 22 14:18:36 2017 -0800

----------------------------------------------------------------------
 .../scheduler/fair/FairScheduler.java           |   8 -
 .../fair/FairSchedulerConfiguration.java        |  16 --
 .../scheduler/fair/FairSchedulerEventLog.java   | 152 -------------------
 .../fair/TestFairSchedulerEventLog.java         |  83 ----------
 4 files changed, 259 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cc9479d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index b2978d4..661d0a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -177,7 +177,6 @@ public class FairScheduler extends
   protected double rackLocalityThreshold; // Cluster threshold for rack locality
   protected long nodeLocalityDelayMs; // Delay for node locality
   protected long rackLocalityDelayMs; // Delay for rack locality
-  private FairSchedulerEventLog eventLog; // Machine-readable event log
   protected boolean assignMultiple; // Allocate multiple containers per
                                     // heartbeat
   @VisibleForTesting
@@ -404,10 +403,6 @@ public class FairScheduler extends
     return continuousSchedulingSleepMs;
   }
 
-  public FairSchedulerEventLog getEventLog() {
-    return eventLog;
-  }
-
   /**
    * Add a new application to the scheduler, with a given id, queue name, and
    * user. This will accept a new app even if the user or queue is above
@@ -875,7 +870,6 @@ public class FairScheduler extends
     try {
       writeLock.lock();
       long start = getClock().getTime();
-      eventLog.log("HEARTBEAT", nm.getHostName());
       super.nodeUpdate(nm);
 
       FSSchedulerNode fsNode = getFSSchedulerNode(nm.getNodeID());
@@ -1284,8 +1278,6 @@ public class FairScheduler extends
 
       // This stores per-application scheduling information
       this.applications = new ConcurrentHashMap<>();
-      this.eventLog = new FairSchedulerEventLog();
-      eventLog.init(this.conf);
 
       allocConf = new AllocationConfiguration(conf);
       try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cc9479d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index 9c9eee6..38e71a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -17,7 +17,6 @@
 */
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
-import java.io.File;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -64,12 +63,6 @@ public class FairSchedulerConfiguration extends Configuration {
   public static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file";
   protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml";
   
-  /** Whether to enable the Fair Scheduler event log */
-  public static final String EVENT_LOG_ENABLED = CONF_PREFIX + "event-log-enabled";
-  public static final boolean DEFAULT_EVENT_LOG_ENABLED = false;
-
-  protected static final String EVENT_LOG_DIR = "eventlog.dir";
-
   /** Whether pools can be created that were not specified in the FS configuration file
    */
   protected static final String ALLOW_UNDECLARED_POOLS = CONF_PREFIX + "allow-undeclared-pools";
@@ -255,15 +248,6 @@ public class FairSchedulerConfiguration extends Configuration {
     return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT);
   }
 
-  public boolean isEventLogEnabled() {
-    return getBoolean(EVENT_LOG_ENABLED, DEFAULT_EVENT_LOG_ENABLED);
-  }
-  
-  public String getEventlogDir() {
-    return get(EVENT_LOG_DIR, new File(System.getProperty("hadoop.log.dir",
-    		"/tmp/")).getAbsolutePath() + File.separator + "fairscheduler");
-  }
-  
   public long getWaitTimeBeforeNextStarvationCheck() {
     return getLong(WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS,
         DEFAULT_WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cc9479d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
deleted file mode 100644
index 803ce78..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.log4j.DailyRollingFileAppender;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
-import org.apache.log4j.spi.LoggingEvent;
-
-/**
- * Event log used by the fair scheduler for machine-readable debug info.
- * This class uses a log4j rolling file appender to write the log, but uses
- * a custom tab-separated event format of the form:
- * <pre>
- * DATE    EVENT_TYPE   PARAM_1   PARAM_2   ...
- * </pre>
- * Various event types are used by the fair scheduler. The purpose of logging
- * in this format is to enable tools to parse the history log easily and read
- * internal scheduler variables, rather than trying to make the log human
- * readable. The fair scheduler also logs human readable messages in the
- * JobTracker's main log.
- *
- * Constructing this class creates a disabled log. It must be initialized
- * using {@link FairSchedulerEventLog#init(FairSchedulerConfiguration)} to
- * begin writing to the file.
- */
-@Private
-@Unstable
-class FairSchedulerEventLog {
-  private static final Log LOG = LogFactory.getLog(FairSchedulerEventLog.class.getName());
-
-  /** Set to true if logging is disabled due to an error. */
-  private boolean logDisabled = true;
-
-  /**
-   * Log directory, set by mapred.fairscheduler.eventlog.location in conf file;
-   * defaults to {hadoop.log.dir}/fairscheduler.
-   */
-  private String logDir;
-
-  /**
-   * Active log file, which is {LOG_DIR}/hadoop-{user}-fairscheduler.log.
-   * Older files are also stored as {LOG_FILE}.date (date format YYYY-MM-DD).
-   */
-  private String logFile;
-
-  /** Log4j appender used to write to the log file */
-  private DailyRollingFileAppender appender;
-
-  boolean init(FairSchedulerConfiguration conf) {
-    if (conf.isEventLogEnabled()) {
-      try {
-        logDir = conf.getEventlogDir();
-        File logDirFile = new File(logDir);
-        if (!logDirFile.exists()) {
-          if (!logDirFile.mkdirs()) {
-            throw new IOException(
-                "Mkdirs failed to create " + logDirFile.toString());
-          }
-        }
-        String username = System.getProperty("user.name");
-        logFile = String.format("%s%shadoop-%s-fairscheduler.log",
-            logDir, File.separator, username);
-        logDisabled = false;
-        PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n");
-        appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd");
-        appender.activateOptions();
-        LOG.info("Initialized fair scheduler event log, logging to " + logFile);
-      } catch (IOException e) {
-        LOG.error(
-            "Failed to initialize fair scheduler event log. Disabling it.", e);
-        logDisabled = true;
-      }
-    } else {
-      logDisabled = true;
-    }
-    return !(logDisabled);
-  }
-
-  /**
-   * Log an event, writing a line in the log file of the form
-   * <pre>
-   * DATE    EVENT_TYPE   PARAM_1   PARAM_2   ...
-   * </pre>
-   */
-  synchronized void log(String eventType, Object... params) {
-    try {
-      if (logDisabled)
-        return;
-      StringBuffer buffer = new StringBuffer();
-      buffer.append(eventType);
-      for (Object param: params) {
-        buffer.append("\t");
-        buffer.append(param);
-      }
-      String message = buffer.toString();
-      Logger logger = Logger.getLogger(getClass());
-      appender.append(new LoggingEvent("", logger, Level.INFO, message, null));
-    } catch (Exception e) {
-      LOG.error("Failed to append to fair scheduler event log", e);
-      logDisabled = true;
-    }
-  }
-
-  /**
-   * Flush and close the log.
-   */
-  synchronized void shutdown() {
-    try {
-      if (appender != null)
-        appender.close();
-    } catch (Exception e) {
-      LOG.error("Failed to close fair scheduler event log", e);
-      logDisabled = true;
-    }
-  }
-
-  synchronized boolean isEnabled() {
-    return !logDisabled;
-  }
-  
-  public String getLogFile() {
-    return logFile;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cc9479d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
deleted file mode 100644
index 09c2370..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestFairSchedulerEventLog {
-  private File logFile;
-  private FairScheduler scheduler;
-  private ResourceManager resourceManager;
-  
-  @Before
-  public void setUp() throws IOException {
-    scheduler = new FairScheduler();
-    
-    Configuration conf = new YarnConfiguration();
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
-        ResourceScheduler.class);
-    conf.set(FairSchedulerConfiguration.EVENT_LOG_ENABLED, "true");
-
-    // All tests assume only one assignment per node update
-    conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
-    resourceManager = new MockRM(conf);
-    ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
-    scheduler.init(conf);
-    scheduler.start();
-    scheduler.reinitialize(conf, resourceManager.getRMContext());
-  }
-
-  /**
-   * Make sure the scheduler creates the event log.
-   */
-  @Test
-  public void testCreateEventLog() throws IOException {
-    FairSchedulerEventLog eventLog = scheduler.getEventLog();
-    
-    logFile = new File(eventLog.getLogFile());
-    Assert.assertTrue(logFile.exists());
-  }
-  
-  @After
-  public void tearDown() {
-    logFile.delete();
-    logFile.getParentFile().delete(); // fairscheduler/
-    if (scheduler != null) {
-      scheduler.stop();
-      scheduler = null;
-    }
-    if (resourceManager != null) {
-      resourceManager.stop();
-      resourceManager = null;
-    }
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org