hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject [01/12] hbase git commit: HBASE-10378 Refactor write-ahead-log implementation -- ADDEDNUM
Date Tue, 18 Nov 2014 20:12:55 GMT
Repository: hbase
Updated Branches:
  refs/heads/master f5e05eb83 -> b94e6738d


http://git-wip-us.apache.org/repos/asf/hbase/blob/b94e6738/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
index 2da631f..e5cd212 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
@@ -40,8 +40,8 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALFactory;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -65,6 +65,7 @@ public class TestMergeTool extends HBaseTestCase {
   private HTableDescriptor desc;
   private byte [][][] rows;
   private MiniDFSCluster dfsCluster = null;
+  private WALFactory wals;
 
   @Before
   public void setUp() throws Exception {
@@ -147,6 +148,7 @@ public class TestMergeTool extends HBaseTestCase {
     // we will end up with a local file system
 
     super.setUp();
+    wals = new WALFactory(conf, null, "TestMergeTool");
     try {
       // Create meta region
       createMetaRegion();
@@ -187,6 +189,7 @@ public class TestMergeTool extends HBaseTestCase {
         HRegion.closeHRegion(r);
       }
     }
+    wals.close();
     TEST_UTIL.shutdownMiniCluster();
   }
 
@@ -200,7 +203,7 @@ public class TestMergeTool extends HBaseTestCase {
    * @throws Exception
    */
   private HRegion mergeAndVerify(final String msg, final String regionName1,
-    final String regionName2, final HLog log, final int upperbound)
+    final String regionName2, final WAL log, final int upperbound)
   throws Exception {
     Merge merger = new Merge(this.conf);
     LOG.info(msg);
@@ -274,39 +277,26 @@ public class TestMergeTool extends HBaseTestCase {
       // Close the region and delete the log
       HRegion.closeHRegion(regions[i]);
     }
-
-    // Create a log that we can reuse when we need to open regions
-    Path logPath = new Path("/tmp");
-    String logName = HConstants.HREGION_LOGDIR_NAME + "_"
-      + System.currentTimeMillis();
-    LOG.info("Creating log " + logPath.toString() + "/" + logName);
-
-    HLog log = HLogFactory.createHLog(this.fs, logPath,
-        logName, this.conf);
-
-    try {
-       // Merge Region 0 and Region 1
-      HRegion merged = mergeAndVerify("merging regions 0 and 1 ",
-        this.sourceRegions[0].getRegionNameAsString(),
-        this.sourceRegions[1].getRegionNameAsString(), log, 2);
-
-      // Merge the result of merging regions 0 and 1 with region 2
-      merged = mergeAndVerify("merging regions 0+1 and 2",
-        merged.getRegionInfo().getRegionNameAsString(),
-        this.sourceRegions[2].getRegionNameAsString(), log, 3);
-
-      // Merge the result of merging regions 0, 1 and 2 with region 3
-      merged = mergeAndVerify("merging regions 0+1+2 and 3",
-        merged.getRegionInfo().getRegionNameAsString(),
-        this.sourceRegions[3].getRegionNameAsString(), log, 4);
-
-      // Merge the result of merging regions 0, 1, 2 and 3 with region 4
-      merged = mergeAndVerify("merging regions 0+1+2+3 and 4",
-        merged.getRegionInfo().getRegionNameAsString(),
-        this.sourceRegions[4].getRegionNameAsString(), log, rows.length);
-    } finally {
-      log.closeAndDelete();
-    }
+    WAL log = wals.getWAL(new byte[]{});
+     // Merge Region 0 and Region 1
+    HRegion merged = mergeAndVerify("merging regions 0 and 1 ",
+      this.sourceRegions[0].getRegionNameAsString(),
+      this.sourceRegions[1].getRegionNameAsString(), log, 2);
+
+    // Merge the result of merging regions 0 and 1 with region 2
+    merged = mergeAndVerify("merging regions 0+1 and 2",
+      merged.getRegionInfo().getRegionNameAsString(),
+      this.sourceRegions[2].getRegionNameAsString(), log, 3);
+
+    // Merge the result of merging regions 0, 1 and 2 with region 3
+    merged = mergeAndVerify("merging regions 0+1+2 and 3",
+      merged.getRegionInfo().getRegionNameAsString(),
+      this.sourceRegions[3].getRegionNameAsString(), log, 4);
+
+    // Merge the result of merging regions 0, 1, 2 and 3 with region 4
+    merged = mergeAndVerify("merging regions 0+1+2+3 and 4",
+      merged.getRegionInfo().getRegionNameAsString(),
+      this.sourceRegions[4].getRegionNameAsString(), log, rows.length);
   }
 
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b94e6738/hbase-shell/src/main/ruby/hbase/admin.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb
index 971df06..1fe496d 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -82,10 +82,12 @@ module Hbase
     end
 
     #----------------------------------------------------------------------------------------------
-    # Requests a regionserver's HLog roll
-    def hlog_roll(server_name)
-      @admin.rollHLogWriter(server_name)
+    # Requests a regionserver's WAL roll
+    def wal_roll(server_name)
+      @admin.rollWALWriter(server_name)
     end
+    # TODO remove older hlog_roll version
+    alias :hlog_roll :wal_roll
 
     #----------------------------------------------------------------------------------------------
     # Requests a table or region split

http://git-wip-us.apache.org/repos/asf/hbase/blob/b94e6738/hbase-shell/src/main/ruby/shell.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb
index a06371a..620033c 100644
--- a/hbase-shell/src/main/ruby/shell.rb
+++ b/hbase-shell/src/main/ruby/shell.rb
@@ -321,13 +321,17 @@ Shell.load_command_group(
     merge_region
     unassign
     zk_dump
-    hlog_roll
+    wal_roll
     catalogjanitor_run
     catalogjanitor_switch
     catalogjanitor_enabled
     compact_rs
     trace
-  ]
+  ],
+  # TODO remove older hlog_roll command
+  :aliases => {
+    'wal_roll' => ['hlog_roll']
+  }
 )
 
 Shell.load_command_group(

http://git-wip-us.apache.org/repos/asf/hbase/blob/b94e6738/hbase-shell/src/main/ruby/shell/commands/hlog_roll.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/hlog_roll.rb b/hbase-shell/src/main/ruby/shell/commands/hlog_roll.rb
deleted file mode 100644
index b292791..0000000
--- a/hbase-shell/src/main/ruby/shell/commands/hlog_roll.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-module Shell
-  module Commands
-    class HlogRoll < Command
-      def help
-        return <<-EOF
-Roll the log writer. That is, start writing log messages to a new file.
-The name of the regionserver should be given as the parameter.  A
-'server_name' is the host, port plus startcode of a regionserver. For
-example: host187.example.com,60020,1289493121758 (find servername in
-master ui or when you do detailed status in shell)
-EOF
-      end
-
-      def command(server_name)
-        format_simple_command do
-          admin.hlog_roll(server_name)
-        end
-      end
-    end
-  end
-end

http://git-wip-us.apache.org/repos/asf/hbase/blob/b94e6738/src/main/docbkx/book.xml
----------------------------------------------------------------------
diff --git a/src/main/docbkx/book.xml b/src/main/docbkx/book.xml
index 06578ca..706936c 100644
--- a/src/main/docbkx/book.xml
+++ b/src/main/docbkx/book.xml
@@ -2136,7 +2136,7 @@ rs.close();
         <section
           xml:id="regionserver.arch.processes.log">
           <title>LogRoller</title>
-          <para>Periodically checks the RegionServer's HLog.</para>
+          <para>Periodically checks the RegionServer's WAL.</para>
         </section>
       </section>
 
@@ -2531,13 +2531,22 @@ rs.close();
           becomes unavailable before the MemStore is flushed, the WAL ensures that the changes
to
           the data can be replayed. If writing to the WAL fails, the entire operation to
modify the
           data fails.</para>
-          <para>HBase uses an implementation of the <link
-              xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/regionserver/wal/HLog.html">HLog</link>
-            interface for the WAL.
-            Usually, there is only one instance of a WAL per RegionServer. The RegionServer
records Puts and Deletes to
-            it, before recording them to the <xref
+          <para>
+            HBase uses an implementation of the <link xlink:href=
+            "http://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/wal/WAL.html"
+            >WAL</link> interface. Usually, there is only one instance of a WAL
per RegionServer.
+            The RegionServer records Puts and Deletes to it, before recording them to the
<xref
               linkend="store.memstore" /> for the affected <xref
-              linkend="store" />.</para>
+              linkend="store" />.
+          </para>
+          <note>
+            <title>The HLog</title>
+            <para>
+              Prior to 2.0, the interface for WALs in HBase was named <classname>HLog</classname>.
+              In 0.94, HLog was the name of the implementation of the WAL. You will likely
find
+              references to the HLog in documentation tailored to these older versions.
+            </para>
+          </note>
           <para>The WAL resides in HDFS in the <filename>/hbase/WALs/</filename>
directory (prior to
             HBase 0.94, they were stored in <filename>/hbase/.logs/</filename>),
with subdirectories per
             region.</para>
@@ -3439,10 +3448,10 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
                   <varname>hbase.regionserver.global.memstore.lowerLimit</varname>.
</para>
             </listitem>
             <listitem>
-              <para>When the number of HLog per region server reaches the value specified
in
+              <para>When the number of WAL per region server reaches the value specified
in
                   <varname>hbase.regionserver.max.logs</varname>, MemStores from
various regions
-                will be flushed out to disk to reduce HLog count. The flush order is based
on time.
-                Regions with the oldest MemStores are flushed first until HLog count drops
below
+                will be flushed out to disk to reduce WAL count. The flush order is based
on time.
+                Regions with the oldest MemStores are flushed first until WAL count drops
below
                   <varname>hbase.regionserver.max.logs</varname>. </para>
             </listitem>
           </orderedlist>

http://git-wip-us.apache.org/repos/asf/hbase/blob/b94e6738/src/main/docbkx/ops_mgt.xml
----------------------------------------------------------------------
diff --git a/src/main/docbkx/ops_mgt.xml b/src/main/docbkx/ops_mgt.xml
index b52457a..0af8f02 100644
--- a/src/main/docbkx/ops_mgt.xml
+++ b/src/main/docbkx/ops_mgt.xml
@@ -318,9 +318,20 @@ ${HBASE_HOME}/bin/hbase org.apache.hadoop.hbase.mapreduce.<replaceable>UtilityNa
 
         <section
           xml:id="hlog_tool.prettyprint">
-          <title><classname>HLogPrettyPrinter</classname></title>
-          <para><classname>HLogPrettyPrinter</classname> is a tool with
configurable options to
-            print the contents of an HLog. </para>
+          <title>WAL Pretty Printer</title>
+          <para>The WAL Pretty Printer is a tool with configurable options to
+            print the contents of a WAL. You can invoke it via the hbase cli with the 'wal'
command.
+          </para>
+          <screen langauge="bourne"> $ ./bin/hbase wal hdfs://example.org:8020/hbase/.logs/example.org,60020,1283516293161/10.10.21.10%3A60020.1283973724012</screen>
+          <note>
+            <title>WAL Printing in older versions of HBase</title>
+            <para>Prior to version 2.0, the WAL Pretty Printer was called the
+              <classname>HLogPrettyPrinter</classname>, after an internal name
for HBase's write
+              ahead log. In those versions, you can pring the contents of a WAL using the
same
+              configuration as above, but with the 'hlog' command.
+            </para>
+            <screen langauge="bourne"> $ ./bin/hbase hlog hdfs://example.org:8020/hbase/.logs/example.org,60020,1283516293161/10.10.21.10%3A60020.1283973724012</screen>
+          </note>
         </section>
 
       </section>
@@ -1968,7 +1979,7 @@ $ for i in `cat conf/regionservers|sort`; do ./bin/graceful_stop.sh
--restart --
         <varlistentry>
           <term><code>source.logEditsRead</code></term>
           <listitem>
-            <para>number of mutations read from HLogs at the replication source</para>
+            <para>number of mutations read from WALs at the replication source</para>
           </listitem>
         </varlistentry>
         <varlistentry>

http://git-wip-us.apache.org/repos/asf/hbase/blob/b94e6738/src/main/docbkx/performance.xml
----------------------------------------------------------------------
diff --git a/src/main/docbkx/performance.xml b/src/main/docbkx/performance.xml
index 78a828f..1757d3f 100644
--- a/src/main/docbkx/performance.xml
+++ b/src/main/docbkx/performance.xml
@@ -690,9 +690,9 @@ admin.createTable(table, splits);
       xml:id="def.log.flush">
       <title> Table Creation: Deferred Log Flush </title>
       <para> The default behavior for Puts using the Write Ahead Log (WAL) is that
-          <classname>HLog</classname> edits will be written immediately. If deferred
log flush is
+          <classname>WAL</classname> edits will be written immediately. If deferred
log flush is
         used, WAL edits are kept in memory until the flush period. The benefit is aggregated
and
-        asynchronous <classname>HLog</classname>- writes, but the potential downside
is that if the
+        asynchronous <classname>WAL</classname>- writes, but the potential downside
is that if the
         RegionServer goes down the yet-to-be-flushed edits are lost. This is safer, however,
than
         not using WAL at all with Puts. </para>
       <para> Deferred log flush can be configured on tables via <link

http://git-wip-us.apache.org/repos/asf/hbase/blob/b94e6738/src/main/docbkx/troubleshooting.xml
----------------------------------------------------------------------
diff --git a/src/main/docbkx/troubleshooting.xml b/src/main/docbkx/troubleshooting.xml
index 6ca4d06..d57bb08 100644
--- a/src/main/docbkx/troubleshooting.xml
+++ b/src/main/docbkx/troubleshooting.xml
@@ -879,7 +879,7 @@ at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:210)
 <filename>/hbase</filename>
      <filename>/.logs</filename>
           <filename>/&lt;RegionServer&gt;</filename>    (RegionServers)
-               <filename>/&lt;HLog&gt;</filename>           (WAL HLog
files for the RegionServer)
+               <filename>/&lt;WAL&gt;</filename>           (WAL files
for the RegionServer)
             </programlisting>
       </para>
       <para>See the <link
@@ -887,11 +887,11 @@ at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:210)
           Guide</link> for other non-shell diagnostic utilities like <code>fsck</code>.
</para>
       <section
         xml:id="trouble.namenode.0size.hlogs">
-        <title>Zero size HLogs with data in them</title>
+        <title>Zero size WALs with data in them</title>
         <para>Problem: when getting a listing of all the files in a region server's
.logs directory,
           one file has a size of 0 but it contains data.</para>
-        <para>Answer: It's an HDFS quirk. A file that's currently being to will appear
to have a
-          size of 0 but once it's closed it will show its true size</para>
+        <para>Answer: It's an HDFS quirk. A file that's currently being written to
will appear to
+          have a size of 0 but once it's closed it will show its true size</para>
       </section>
       <section
         xml:id="trouble.namenode.uncompaction">
@@ -1583,7 +1583,7 @@ WARN HDFS.DFSClient: DataStreamer Exception: java.io.IOException: Unable
to crea
           <para>If the operation was an attempt to split the log, the following type
of message may
             also appear:</para>
           <screen>
-FATAL wal.HLogSplitter: WriterThread-xxx Got while writing log entry to log            
+FATAL wal.WALSplitter: WriterThread-xxx Got while writing log entry to log            
           </screen>
         </listitem>
       </varlistentry>


Mime
View raw message