hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject svn commit: r1485872 - in /hbase/branches/0.94/src: main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/backup/ main/java/org/apache/hadoop/hbase/regionserver/wal/ test/java/org/apache/hadoop/hbase/
Date Thu, 23 May 2013 22:08:08 GMT
Author: tedyu
Date: Thu May 23 22:08:08 2013
New Revision: 1485872

URL: http://svn.apache.org/r1485872
Log:
HBASE-8522 Archived hfiles and old hlogs may be deleted immediately by HFileCleaner, LogCleaner
in HMaster (Shaohui)


Modified:
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
    hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java?rev=1485872&r1=1485871&r2=1485872&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java Thu May
23 22:08:08 2013
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hbase.regionserver.wal.HLogFileSystem;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Threads;
 
@@ -252,4 +253,15 @@ public abstract class HBaseFileSystem {
     LOG.info(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier);
     Threads.sleep(baseSleepBeforeRetries * sleepMultiplier);
   }
+  
+  /**
+   * rename the src path to dest path and set the dest path's modify time to current timestamp
+   */
+  public static boolean renameAndSetModifyTime(final FileSystem fs, Path src, Path dest)
+      throws IOException {
+    if (!renameDirForFileSystem(fs, src, dest)) return false;
+    // set the modify time for TimeToLive Cleaner
+    fs.setTimes(dest, EnvironmentEdgeManager.currentTimeMillis(), -1);
+    return true;
+  }
 }

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java?rev=1485872&r1=1485871&r2=1485872&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java Thu
May 23 22:08:08 2013
@@ -386,7 +386,7 @@ public class HFileArchiver {
 
       // move the archive file to the stamped backup
       Path backedupArchiveFile = new Path(archiveDir, filename + SEPARATOR + archiveStartTime);
-      if (!HBaseFileSystem.renameDirForFileSystem(fs, archiveFile, backedupArchiveFile))
{
+      if (!HBaseFileSystem.renameAndSetModifyTime(fs, archiveFile, backedupArchiveFile))
{
         LOG.error("Could not rename archive file to backup: " + backedupArchiveFile
             + ", deleting existing file in favor of newer.");
         // try to delete the exisiting file, if we can't rename it
@@ -610,7 +610,7 @@ public class HFileArchiver {
     public boolean moveAndClose(Path dest) throws IOException {
       this.close();
       Path p = this.getPath();
-      return HBaseFileSystem.renameDirForFileSystem(fs, p, dest);
+      return HBaseFileSystem.renameAndSetModifyTime(fs, p, dest);
     }
 
     /**

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1485872&r1=1485871&r2=1485872&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Thu
May 23 22:08:08 2013
@@ -904,7 +904,7 @@ public class HLog implements Syncable {
         i.preLogArchive(p, newPath);
       }
     }
-    if (!HBaseFileSystem.renameDirForFileSystem(fs, p, newPath)) {
+    if (!HBaseFileSystem.renameAndSetModifyTime(this.fs, p, newPath)) {
       throw new IOException("Unable to rename " + p + " to " + newPath);
     }
     // Tell our listeners that a log has been archived.
@@ -966,8 +966,7 @@ public class HLog implements Syncable {
           i.preLogArchive(file.getPath(), p);
         }
       }
-
-      if (!HBaseFileSystem.renameDirForFileSystem(fs, file.getPath(), p)) {
+      if (!HBaseFileSystem.renameAndSetModifyTime(fs, file.getPath(), p)) {
         throw new IOException("Unable to rename " + file.getPath() + " to " + p);
       }
       // Tell our listeners that a log was archived.

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java?rev=1485872&r1=1485871&r2=1485872&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
Thu May 23 22:08:08 2013
@@ -629,7 +629,7 @@ public class HLogSplitter {
     for (Path p : processedLogs) {
       Path newPath = HLog.getHLogArchivePath(oldLogDir, p);
       if (fs.exists(p)) {
-        if (!HBaseFileSystem.renameDirForFileSystem(fs, p, newPath)) {
+        if (!HBaseFileSystem.renameAndSetModifyTime(fs, p, newPath)) {
           LOG.warn("Unable to move  " + p + " to " + newPath);
         } else {
           LOG.debug("Archived processed log " + p + " to " + newPath);

Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java?rev=1485872&r1=1485871&r2=1485872&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java Thu
May 23 22:08:08 2013
@@ -17,10 +17,12 @@
  */
 package org.apache.hadoop.hbase;
 
+import static org.junit.Assert.*;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.net.URI;
+import java.util.UUID;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -31,6 +33,9 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
 import org.apache.hadoop.util.Progressable;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -91,7 +96,40 @@ public class TestHBaseFileSystem {
     assertTrue("Couldn't delete the directory", result);
     fs.delete(rootDir, true);
   }
+  
+  @Test
+  public void testRenameAndSetModifyTime() throws Exception {
+    assertTrue(FSUtils.isHDFS(conf));
+
+    FileSystem fs = FileSystem.get(conf);
+    Path testDir = TEST_UTIL.getDataTestDir("testArchiveFile");
+
+    String file = UUID.randomUUID().toString();
+    Path p = new Path(testDir, file);
+
+    FSDataOutputStream out = fs.create(p);
+    out.close();
+    assertTrue("The created file should be present", FSUtils.isExists(fs, p));
+
+    long expect = System.currentTimeMillis() + 1000;
+    assertFalse(expect == fs.getFileStatus(p).getModificationTime());
 
+    ManualEnvironmentEdge mockEnv = new ManualEnvironmentEdge();
+    mockEnv.setValue(expect);
+    EnvironmentEdgeManager.injectEdge(mockEnv);
+
+    String dstFile = UUID.randomUUID().toString();
+    Path dst = new Path(testDir , dstFile);
+
+    assertTrue(HBaseFileSystem.renameAndSetModifyTime(fs, p, dst));
+    assertFalse("The moved file should not be present", FSUtils.isExists(fs,
+                                                                         p));
+    assertTrue("The dst file should be present", FSUtils.isExists(fs, dst));
+
+    assertEquals(expect, fs.getFileStatus(dst).getModificationTime());
+  }
+
+  
   static class MockFileSystemForCreate extends MockFileSystem {
     @Override
     public boolean exists(Path path) {



Mime
View raw message