Return-Path: Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: (qmail 81832 invoked from network); 21 May 2010 18:49:40 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 21 May 2010 18:49:40 -0000 Received: (qmail 11589 invoked by uid 500); 21 May 2010 18:49:40 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 11553 invoked by uid 500); 21 May 2010 18:49:39 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 11545 invoked by uid 99); 21 May 2010 18:49:39 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 21 May 2010 18:49:39 +0000 X-ASF-Spam-Status: No, hits=-1312.6 required=10.0 tests=ALL_TRUSTED,AWL X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 21 May 2010 18:49:38 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 27D952388903; Fri, 21 May 2010 18:49:18 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r947127 - in /hbase/trunk: ./ src/main/java/org/apache/hadoop/hbase/master/ src/main/java/org/apache/hadoop/hbase/regionserver/ src/main/java/org/apache/hadoop/hbase/regionserver/wal/ src/test/java/org/apache/hadoop/hbase/master/ src/test/j... Date: Fri, 21 May 2010 18:49:17 -0000 To: commits@hbase.apache.org From: jdcryans@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100521184918.27D952388903@eris.apache.org> Author: jdcryans Date: Fri May 21 18:49:17 2010 New Revision: 947127 URL: http://svn.apache.org/viewvc?rev=947127&view=rev Log: HBASE-2580 Make the hlog file names unique Modified: hbase/trunk/CHANGES.txt hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java Modified: hbase/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=947127&r1=947126&r2=947127&view=diff ============================================================================== --- hbase/trunk/CHANGES.txt (original) +++ hbase/trunk/CHANGES.txt Fri May 21 18:49:17 2010 @@ -626,6 +626,7 @@ Release 0.21.0 - Unreleased HBASE-2577 Remove 'core' maven module; move core up a level HBASE-2587 Coral where tests write data when running and make sure clean target removes all written + HBASE-2580 Make the hlog file names unique NEW FEATURES HBASE-1961 HBase EC2 scripts Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java?rev=947127&r1=947126&r2=947127&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java Fri May 21 18:49:17 2010 @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Chore; import org.apache.hadoop.hbase.RemoteExceptionHandler; +import org.apache.hadoop.hbase.regionserver.wal.HLog; import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; @@ -48,8 +49,6 @@ public class OldLogsCleaner extends Chor private final Path oldLogDir; private final LogCleanerDelegate logCleaner; private final Configuration conf; - // We expect a file looking like hlog.dat.ts - private final Pattern pattern = Pattern.compile("\\d*\\.hlog\\.dat\\.\\d*"); /** * @@ -92,7 +91,7 @@ public class OldLogsCleaner extends Chor int nbDeletedLog = 0; for (FileStatus file : files) { Path filePath = file.getPath(); - if (pattern.matcher(filePath.getName()).matches()) { + if (HLog.validateHLogFilename(filePath.getName())) { if (logCleaner.isLogDeletable(filePath) ) { this.fs.delete(filePath, true); nbDeletedLog++; Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java?rev=947127&r1=947126&r2=947127&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java Fri May 21 18:49:17 2010 @@ -40,12 +40,13 @@ public class TimeToLiveLogCleaner implem public boolean isLogDeletable(Path filePath) { long time = 0; long currentTime = System.currentTimeMillis(); - System.out.println(filePath.getName()); String[] parts = filePath.getName().split("\\."); try { - time = Long.parseLong(parts[3]); + time = Long.parseLong(parts[parts.length-1]); } catch (NumberFormatException e) { - e.printStackTrace(); + LOG.error("Unable to parse the timestamp in " + filePath.getName() + + ", deleting it since it's invalid and may not be a hlog", e); + return true; } long life = currentTime - time; if (life < 0) { Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=947127&r1=947126&r2=947127&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Fri May 21 18:49:17 2010 @@ -939,7 +939,8 @@ public class HRegionServer implements HC // instantiate protected HLog instantiateHLog(Path logdir, Path oldLogDir) throws IOException { - HLog newlog = new HLog(fs, logdir, oldLogDir, conf, hlogRoller); + HLog newlog = new HLog(fs, logdir, oldLogDir, conf, hlogRoller, null, + serverInfo.getServerAddress().toString()); return newlog; } Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=947127&r1=947126&r2=947127&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Fri May 21 18:49:17 2010 @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.lang.reflect.Method; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; @@ -45,6 +46,7 @@ import java.util.concurrent.atomic.Atomi import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -123,6 +125,7 @@ public class HLog implements HConstants, private final long optionalFlushInterval; private final long blocksize; private final int flushlogentries; + private final String prefix; private final AtomicInteger unflushedEntries = new AtomicInteger(0); private final Path oldLogDir; private final List actionListeners = @@ -209,6 +212,11 @@ public class HLog implements HConstants, */ private final LogSyncer logSyncerThread; + /** + * Pattern used to validate a HLog file name + */ + private static final Pattern pattern = Pattern.compile(".*\\.\\d*"); + static byte [] COMPLETE_CACHE_FLUSH; static { try { @@ -262,7 +270,7 @@ public class HLog implements HConstants, public HLog(final FileSystem fs, final Path dir, final Path oldLogDir, final Configuration conf, final LogRollListener listener) throws IOException { - this(fs, dir, oldLogDir, conf, listener, null); + this(fs, dir, oldLogDir, conf, listener, null, null); } /** @@ -278,11 +286,14 @@ public class HLog implements HConstants, * @param conf configuration to use * @param listener listerner used to request log rolls * @param actionListener optional listener for hlog actions like archiving + * @param prefix should always be hostname and port in distributed env and + * it will be URL encoded before being used. + * If prefix is null, "hlog" will be used * @throws IOException */ public HLog(final FileSystem fs, final Path dir, final Path oldLogDir, final Configuration conf, final LogRollListener listener, - final LogActionsListener actionListener) + final LogActionsListener actionListener, final String prefix) throws IOException { super(); this.fs = fs; @@ -316,6 +327,9 @@ public class HLog implements HConstants, if (actionListener != null) { addLogActionsListerner(actionListener); } + // If prefix is null||empty then just name it hlog + this.prefix = prefix == null || prefix.isEmpty() ? + "hlog" : URLEncoder.encode(prefix, "UTF8"); // rollWriter sets this.hdfs_out if it can. rollWriter(); @@ -414,7 +428,7 @@ public class HLog implements HConstants, // Clean up current writer. Path oldFile = cleanupCurrentWriter(this.filenum); this.filenum = System.currentTimeMillis(); - Path newPath = computeFilename(this.filenum); + Path newPath = computeFilename(); this.writer = createWriter(fs, newPath, HBaseConfiguration.create(conf)); this.initialReplication = fs.getFileStatus(newPath).getReplication(); @@ -627,7 +641,7 @@ public class HLog implements HConstants, throw e; } if (currentfilenum >= 0) { - oldFile = computeFilename(currentfilenum); + oldFile = computeFilename(); this.outputfiles.put(Long.valueOf(this.logSeqNum.get() - 1), oldFile); } } @@ -650,12 +664,13 @@ public class HLog implements HConstants, /** * This is a convenience method that computes a new filename with a given * file-number. - * @param fn * @return Path */ - public Path computeFilename(final long fn) { - if (fn < 0) return null; - return new Path(dir, HLOG_DATFILE + fn); + protected Path computeFilename() { + if (filenum < 0) { + throw new RuntimeException("hlog file number can't be < 0"); + } + return new Path(dir, prefix + "." + filenum); } /** @@ -1664,8 +1679,12 @@ public class HLog implements HConstants, return dirName.toString(); } + public static boolean validateHLogFilename(String filename) { + return pattern.matcher(filename).matches(); + } + private static Path getHLogArchivePath(Path oldLogDir, Path p) { - return new Path(oldLogDir, System.currentTimeMillis() + "." + p.getName()); + return new Path(oldLogDir, p.getName()); } private static void usage() { Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java?rev=947127&r1=947126&r2=947127&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java Fri May 21 18:49:17 2010 @@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.master; import static org.junit.Assert.assertEquals; +import org.apache.hadoop.fs.FileStatus; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -34,6 +35,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.conf.Configuration; +import java.net.URLEncoder; import java.util.concurrent.atomic.AtomicBoolean; public class TestOldLogsCleaner { @@ -74,32 +76,41 @@ public class TestOldLogsCleaner { Configuration c = TEST_UTIL.getConfiguration(); Path oldLogDir = new Path(TEST_UTIL.getTestDir(), HConstants.HREGION_OLDLOGDIR_NAME); + String fakeMachineName = URLEncoder.encode("regionserver:60020", "UTF8"); FileSystem fs = FileSystem.get(c); AtomicBoolean stop = new AtomicBoolean(false); OldLogsCleaner cleaner = new OldLogsCleaner(1000, stop,c, fs, oldLogDir); + // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files long now = System.currentTimeMillis(); fs.delete(oldLogDir, true); fs.mkdirs(oldLogDir); fs.createNewFile(new Path(oldLogDir, "a")); - fs.createNewFile(new Path(oldLogDir, "1.hlog.dat.a")); - fs.createNewFile(new Path(oldLogDir, "1.hlog.dat." + now)); - for(int i = 0; i < 30; i++) { - fs.createNewFile(new Path(oldLogDir, 1 + "hlog.dat." + - (now - 6000000 - i))); + fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + "a")); + fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + now)); + System.out.println("Now is: " + now); + for (int i = 0; i < 30; i++) { + fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + (now - 6000000 - i) )); } - fs.createNewFile(new Path(oldLogDir, "a.hlog.dat." + (now + 10000))); + for (FileStatus stat : fs.listStatus(oldLogDir)) { + System.out.println(stat.getPath().toString()); + } + + fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + (now + 10000) )); assertEquals(34, fs.listStatus(oldLogDir).length); + // This will take care of 20 old log files (default max we can delete) cleaner.chore(); assertEquals(14, fs.listStatus(oldLogDir).length); + // We will delete all remaining log files and those that are invalid cleaner.chore(); - assertEquals(1, fs.listStatus(oldLogDir).length); + // We end up with the current log file and a newer one + assertEquals(2, fs.listStatus(oldLogDir).length); } } Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java?rev=947127&r1=947126&r2=947127&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java Fri May 21 18:49:17 2010 @@ -150,7 +150,7 @@ public class TestHLog extends HBaseTestC // gives you EOFE. wal.sync(); // Open a Reader. - Path walPath = wal.computeFilename(wal.getFilenum()); + Path walPath = wal.computeFilename(); HLog.Reader reader = HLog.getReader(fs, walPath, conf); int count = 0; HLog.Entry entry = new HLog.Entry(); @@ -281,7 +281,7 @@ public class TestHLog extends HBaseTestC long logSeqId = log.startCacheFlush(); log.completeCacheFlush(regionName, tableName, logSeqId, info.isMetaRegion()); log.close(); - Path filename = log.computeFilename(log.getFilenum()); + Path filename = log.computeFilename(); log = null; // Now open a reader on the log and assert append worked. reader = HLog.getReader(fs, filename, conf); @@ -349,7 +349,7 @@ public class TestHLog extends HBaseTestC long logSeqId = log.startCacheFlush(); log.completeCacheFlush(hri.getRegionName(), tableName, logSeqId, false); log.close(); - Path filename = log.computeFilename(log.getFilenum()); + Path filename = log.computeFilename(); log = null; // Now open a reader on the log and assert append worked. reader = HLog.getReader(fs, filename, conf); Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java?rev=947127&r1=947126&r2=947127&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java Fri May 21 18:49:17 2010 @@ -84,7 +84,7 @@ public class TestLogActionsListener { public void testActionListener() throws Exception { DummyLogActionsListener list = new DummyLogActionsListener(); DummyLogActionsListener laterList = new DummyLogActionsListener(); - HLog hlog = new HLog(fs, logDir, oldLogDir, conf, null, list); + HLog hlog = new HLog(fs, logDir, oldLogDir, conf, null, list, null); HRegionInfo hri = new HRegionInfo(new HTableDescriptor(SOME_BYTES), SOME_BYTES, SOME_BYTES, false);