Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@hbase.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r945963 - in /hbase/trunk: ./
 core/src/main/java/org/apache/hadoop/hbase/master/
 core/src/test/java/org/apache/hadoop/hbase/master/
Date: Tue, 18 May 2010 23:40:50 -0000
To: commits@hbase.apache.org
From: jdcryans@apache.org
Message-Id: <20100518234050.64A0C23888FE@eris.apache.org>

Author: jdcryans
Date: Tue May 18 23:40:49 2010
New Revision: 945963

URL: http://svn.apache.org/viewvc?rev=945963&view=rev
Log:
HBASE-2529  Make OldLogsCleaner easier to extend

Added:
    hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java
    hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
Modified:
    hbase/trunk/CHANGES.txt
    hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java
    hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java

Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=945963&r1=945962&r2=945963&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Tue May 18 23:40:49 2010
@@ -641,6 +641,7 @@ Release 0.21.0 - Unreleased
    HBASE-2438  Addition of a Column Pagination Filter (Paul Kist via Stack)
    HBASE-2473  Add to admin create table start and end key params and
                desired number of regions
+   HBASE-2529  Make OldLogsCleaner easier to extend
 
   OPTIMIZATIONS
    HBASE-410   [testing] Speed up the test suite

Added: hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java
URL: http://svn.apache.org/viewvc/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java?rev=945963&view=auto
==============================================================================
--- hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java (added)
+++ hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java Tue May 18 23:40:49 2010
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Interface for the log cleaning function inside the master. Only 1 is called
+ * so if the desired effect is the mix of many cleaners, do call them yourself
+ * in order to control the flow.
+ * HBase ships with OldLogsCleaner as the default implementation
+ */
+public interface LogCleanerDelegate extends Configurable {
+
+  /**
+   * Should the master delete the log or keep it?
+   * @param filePath full path to log.
+   * @return true if the log is deletable, false if not
+   */
+  public boolean isLogDeletable(Path filePath);
+}

Modified: hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java?rev=945963&r1=945962&r2=945963&view=diff
==============================================================================
--- hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java (original)
+++ hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java Tue May 18 23:40:49 2010
@@ -41,59 +41,59 @@ public class OldLogsCleaner extends Chor
 
   static final Log LOG = LogFactory.getLog(OldLogsCleaner.class.getName());
 
-  // Configured time a log can be kept after it was closed
-  private final long ttl;
   // Max number we can delete on every chore, this is to make sure we don't
   // issue thousands of delete commands around the same time
   private final int maxDeletedLogs;
   private final FileSystem fs;
   private final Path oldLogDir;
-  // We expect a file looking like ts.hlog.dat.ts
+  private final LogCleanerDelegate logCleaner;
+  private final Configuration conf;
+  // We expect a file looking like hlog.dat.ts
   private final Pattern pattern = Pattern.compile("\\d*\\.hlog\\.dat\\.\\d*");
 
   /**
    *
-   * @param p
-   * @param s
-   * @param conf
-   * @param fs
-   * @param oldLogDir
+   * @param p the period of time to sleep between each run
+   * @param s the stopper boolean
+   * @param conf configuration to use
+   * @param fs handle to the FS
+   * @param oldLogDir the path to the archived logs
    */
   public OldLogsCleaner(final int p, final AtomicBoolean s,
                         Configuration conf, FileSystem fs,
                         Path oldLogDir) {
     super(p, s);
-    this.ttl = conf.getLong("hbase.master.logcleaner.ttl", 600000);
     this.maxDeletedLogs =
         conf.getInt("hbase.master.logcleaner.maxdeletedlogs", 20);
     this.fs = fs;
     this.oldLogDir = oldLogDir;
+    this.conf = conf;
+    this.logCleaner = getLogCleaner();
+  }
+
+  private LogCleanerDelegate getLogCleaner() {
+    try {
+      Class c = Class.forName(conf.get("hbase.master.logcleanerplugin.impl",
+        TimeToLiveLogCleaner.class.getCanonicalName()));
+      LogCleanerDelegate cleaner = (LogCleanerDelegate) c.newInstance();
+      cleaner.setConf(conf);
+      return cleaner;
+    } catch (Exception e) {
+      LOG.warn("Passed log cleaner implementation throws errors, " +
+          "defaulting to TimeToLiveLogCleaner", e);
+      return new TimeToLiveLogCleaner();
+    }
   }
 
   @Override
   protected void chore() {
     try {
       FileStatus[] files = this.fs.listStatus(this.oldLogDir);
-      long currentTime = System.currentTimeMillis();
       int nbDeletedLog = 0;
       for (FileStatus file : files) {
         Path filePath = file.getPath();
-
         if (pattern.matcher(filePath.getName()).matches()) {
-          String[] parts = filePath.getName().split("\\.");
-          long time = 0;
-          try {
-            time = Long.parseLong(parts[3]);
-          } catch (NumberFormatException e) {
-            // won't happen
-          }
-          long life = currentTime - time;
-          if (life < 0) {
-            LOG.warn("Found a log newer than current time, " +
-                "probably a clock skew");
-            continue;
-          }
-          if (life > ttl) {
+          if (logCleaner.isLogDeletable(filePath) ) {
             this.fs.delete(filePath, true);
             nbDeletedLog++;
           }
@@ -106,7 +106,6 @@ public class OldLogsCleaner extends Chor
         if (nbDeletedLog >= maxDeletedLogs) {
           break;
         }
-
       }
     } catch (IOException e) {
       e = RemoteExceptionHandler.checkIOException(e);

Added: hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java?rev=945963&view=auto
==============================================================================
--- hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java (added)
+++ hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java Tue May 18 23:40:49 2010
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Log cleaner that uses the timestamp of the hlog to determine if it should
+ * be deleted. By default they are allowed to live for 10 minutes.
+ */
+public class TimeToLiveLogCleaner implements LogCleanerDelegate {
+
+  static final Log LOG =
+      LogFactory.getLog(TimeToLiveLogCleaner.class.getName());
+  private Configuration conf;
+  // Configured time a log can be kept after it was closed
+  private long ttl;
+
+  @Override
+  public boolean isLogDeletable(Path filePath) {
+    long time = 0;
+    long currentTime = System.currentTimeMillis();
+    System.out.println(filePath.getName());
+    String[] parts = filePath.getName().split("\\.");
+    try {
+      time = Long.parseLong(parts[3]);
+    } catch (NumberFormatException e) {
+      e.printStackTrace();
+    }
+    long life = currentTime - time;
+    if (life < 0) {
+      LOG.warn("Found a log newer than current time, " +
+          "probably a clock skew");
+      return false;
+    }
+    return life > ttl;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+    this.ttl = conf.getLong("hbase.master.logcleaner.ttl", 600000);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+}

Modified: hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java?rev=945963&r1=945962&r2=945963&view=diff
==============================================================================
--- hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java (original)
+++ hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java Tue May 18 23:40:49 2010
@@ -86,9 +86,10 @@ public class TestOldLogsCleaner {
     fs.createNewFile(new Path(oldLogDir, "1.hlog.dat.a"));
     fs.createNewFile(new Path(oldLogDir, "1.hlog.dat." + now));
     for(int i = 0; i < 30; i++) {
-      fs.createNewFile(new Path(oldLogDir, i + ".hlog.dat." +(now - 6000000)));
+      fs.createNewFile(new Path(oldLogDir, 1 + "hlog.dat." +
+          (now - 6000000 - i)));
     }
-    fs.createNewFile(new Path(oldLogDir, "a.hlog.dat." +(now + 10000)));
+    fs.createNewFile(new Path(oldLogDir, "a.hlog.dat." + (now + 10000)));
 
     assertEquals(34, fs.listStatus(oldLogDir).length);