hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject svn commit: r1485870 - in /hbase/trunk/hbase-server/src: main/java/org/apache/hadoop/hbase/backup/ main/java/org/apache/hadoop/hbase/regionserver/wal/ main/java/org/apache/hadoop/hbase/util/ test/java/org/apache/hadoop/hbase/util/
Date Thu, 23 May 2013 22:04:37 GMT
Author: tedyu
Date: Thu May 23 22:04:36 2013
New Revision: 1485870

URL: http://svn.apache.org/r1485870
Log:
HBASE-8522 Archived hfiles and old hlogs may be deleted immediately by HFileCleaner, LogCleaner
in HMaster (Liu Shaohui)


Modified:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java?rev=1485870&r1=1485869&r2=1485870&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
Thu May 23 22:04:36 2013
@@ -582,7 +582,7 @@ public class HFileArchiver {
     public boolean moveAndClose(Path dest) throws IOException {
       this.close();
       Path p = this.getPath();
-      return fs.rename(p, dest);
+      return FSUtils.renameAndSetModifyTime(fs, p, dest);
     }
 
     /**

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java?rev=1485870&r1=1485869&r2=1485870&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
Thu May 23 22:04:36 2013
@@ -701,7 +701,7 @@ class FSHLog implements HLog, Syncable {
         i.preLogArchive(p, newPath);
       }
     }
-    if (!this.fs.rename(p, newPath)) {
+    if (!FSUtils.renameAndSetModifyTime(this.fs, p, newPath)) {
       throw new IOException("Unable to rename " + p + " to " + newPath);
     }
     // Tell our listeners that a log has been archived.
@@ -754,7 +754,7 @@ class FSHLog implements HLog, Syncable {
           }
         }
 
-        if (!fs.rename(file.getPath(),p)) {
+        if (!FSUtils.renameAndSetModifyTime(fs, file.getPath(), p)) {
           throw new IOException("Unable to rename " + file.getPath() + " to " + p);
         }
         // Tell our listeners that a log was archived.

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java?rev=1485870&r1=1485869&r2=1485870&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
Thu May 23 22:04:36 2013
@@ -672,7 +672,7 @@ public class HLogSplitter {
     for (Path p : processedLogs) {
       Path newPath = FSHLog.getHLogArchivePath(oldLogDir, p);
       if (fs.exists(p)) {
-        if (!fs.rename(p, newPath)) {
+        if (!FSUtils.renameAndSetModifyTime(fs, p, newPath)) {
           LOG.warn("Unable to move  " + p + " to " + newPath);
         } else {
           LOG.debug("Archived processed log " + p + " to " + newPath);

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=1485870&r1=1485869&r2=1485870&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Thu May
23 22:04:36 2013
@@ -1641,4 +1641,12 @@ public abstract class FSUtils {
       }
     }
   }
+
+  public static boolean renameAndSetModifyTime(final FileSystem fs, Path src, Path dest)
+      throws IOException {
+    if (!fs.rename(src, dest)) return false;
+    // set the modify time for TimeToLive Cleaner
+    fs.setTimes(dest, EnvironmentEdgeManager.currentTimeMillis(), -1);
+    return true;
+  }
 }

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java?rev=1485870&r1=1485869&r2=1485870&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java Thu
May 23 22:04:36 2013
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertFal
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotEquals;
 
 import java.io.File;
 import java.io.IOException;
@@ -287,4 +288,39 @@ public class TestFSUtils {
     }
   }
 
+  @Test
+  public void testRenameAndSetModifyTime() throws Exception {
+    HBaseTestingUtility htu = new HBaseTestingUtility();
+    Configuration conf = htu.getConfiguration();
+        
+    MiniDFSCluster cluster = htu.startMiniDFSCluster(1);
+    assertTrue(FSUtils.isHDFS(conf));
+
+    FileSystem fs = FileSystem.get(conf);
+    Path testDir = htu.getDataTestDir("testArchiveFile");
+    
+    String file = UUID.randomUUID().toString();
+    Path p = new Path(testDir, file);
+
+    FSDataOutputStream out = fs.create(p);
+    out.close();
+    assertTrue("The created file should be present", FSUtils.isExists(fs, p));
+    
+    long expect = System.currentTimeMillis() + 1000;
+    assertNotEquals(expect, fs.getFileStatus(p).getModificationTime());
+    
+    ManualEnvironmentEdge mockEnv = new ManualEnvironmentEdge();
+    mockEnv.setValue(expect);
+    EnvironmentEdgeManager.injectEdge(mockEnv);
+    
+    String dstFile = UUID.randomUUID().toString();
+    Path dst = new Path(testDir , dstFile);
+    
+    assertTrue(FSUtils.renameAndSetModifyTime(fs, p, dst));
+    assertFalse("The moved file should not be present", FSUtils.isExists(fs, p));
+    assertTrue("The dst file should be present", FSUtils.isExists(fs, dst));
+
+    assertEquals(expect, fs.getFileStatus(dst).getModificationTime());
+    cluster.shutdown();
+  }
 }



Mime
View raw message