hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject svn commit: r891871 - in /hadoop/hbase/branches/0.20_on_hadoop-0.18.3: ./ bin/ conf/ src/contrib/ec2/ src/contrib/ec2/bin/ src/contrib/ec2/bin/image/ src/java/ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/client/ src/java/org/apac...
Date Thu, 17 Dec 2009 20:26:31 GMT
Author: apurtell
Date: Thu Dec 17 20:26:29 2009
New Revision: 891871

URL: http://svn.apache.org/viewvc?rev=891871&view=rev
Log:
pull up to 0.20 branch head (0.20.3)

Added:
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/functions.sh
Modified:
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/HBase.rb
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/hirb.rb
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/loadtable.rb
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/conf/hbase-default.xml
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/README.txt
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/create-hbase-image
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/hbase-ec2-env.sh
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/hbase-ec2-init-remote.sh
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/image/create-hbase-image-remote
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-master
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-zookeeper
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HBaseConfiguration.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Scan.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/MurmurHash.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/overview.html
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
    hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/webapps/master/master.jsp

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt Thu Dec 17 20:26:29 2009
@@ -1,6 +1,51 @@
 HBase Change Log
 
-Release 0.20.2 - Unreleased
+Release 0.20.3 - Unreleased
+  INCOMPATIBLE CHANGES
+
+  BUG FIXES
+   HBASE-1999  When HTable goes away, close zk session in shutdown hook or
+               something...
+   HBASE-1997  zk tick time bounds maximum zk session time
+   HBASE-1979  MurmurHash does not yield the same results as the reference
+               C++ implementation when size % 4 >= 2 (Olivier Gillet via Stack)
+   HBASE-1994  Master will lose hlog entries while splitting if region has
+               empty oldlogfile.log (Lars George via Stack)
+   HBASE-2022  NPE in housekeeping kills RS
+   HBASE-2033  Shell scan 'limit' is off by one
+   HBASE-2034  [Bulk load tools] loadtable.rb calls an undefined method
+               `descendingIterator' (Ching-Shen Chen via Stack)
+   HBASE-2043  Shell's scan broken
+   HBASE-2047  Example command in the "Getting Started" 
+               documentation doesn't work (Benoit Sigoure via JD)
+   HBASE-2048  Small inconsistency in the "Example API Usage"
+               (Benoit Sigoure via JD)
+
+  IMPROVEMENTS
+   HBASE-1961  HBase EC2 scripts
+   HBASE-1970  Export does one version only; make it configurable how many
+               it does
+   HBASE-1975  SingleColumnValueFilter: Add ability to match the value of
+               previous versions of the specified column
+               (Jeremiah Jacquet via Stack)
+   HBASE-1987  The Put object has no simple read methods for checking what
+               has already been added (Ryan Smith via Stack)
+   HBASE-2009  [EC2] Support mapreduce
+   HBASE-2012  [EC2] LZO support
+   HBASE-2011  Add zktop like output to HBase's master UI (Lars George via
+               Andrew Purtell)
+   HBASE-2018  Updates to .META. blocked under high MemStore load
+   HBASE-2029  Reduce shell exception dump on console
+               (Lars George and J-D via Stack)
+   HBASE-2027  HConnectionManager.HBASE_INSTANCES leaks TableServers
+               (Dave Latham via Stack)
+   HBASE-2031  When starting HQuorumPeer, try to match on more than 1 address
+   HBASE-2046  Remove CompactionLimitThread
+   HBASE-2052  Make hbase more 'live' when comes to noticing table creation, 
+               splits, etc., for 0.20.3
+   HBASE-2049  Cleanup HLog binary log output (Dave Latham via Stack)
+
+Release 0.20.2 - November 18th, 2009
   INCOMPATIBLE CHANGES
 
   BUG FIXES
@@ -18,7 +63,8 @@
    HBASE-1777  column length is not checked before saved to memstore
    HBASE-1895  HConstants.MAX_ROW_LENGTH is incorrectly 64k, should be 32k
    HBASE-1925  IllegalAccessError: Has not been initialized (getMaxSequenceId)
-   HBASE-1929  If hbase-default.xml is not in CP, zk session timeout is 10 secs!
+   HBASE-1929  If hbase-default.xml is not in CP, zk session timeout is 10
+               seconds
    HBASE-1927  Scanners not closed properly in certain circumstances
    HBASE-1934  NullPointerException in ClientScanner (Andrew Purtell via Stack)
    HBASE-1946  Unhandled exception at regionserver (Dmitriy Lyfar via Stack)
@@ -27,7 +73,8 @@
                (Andrew McCall via Clint Morgan and Stack)
    HBASE-1953  Overhaul of overview.html (html fixes, typos, consistency) -
                no content changes (Lars Francke via Stack)
-   HBASE-1954  Transactional scans do not see newest put (Clint Morgan via Stack)
+   HBASE-1954  Transactional scans do not see newest put (Clint Morgan via
+               Stack)
    HBASE-1919  code: HRS.delete seems to ignore exceptions it shouldnt
    HBASE-1951  Stack overflow when calling HTable.checkAndPut() 
                when deleting a lot of values
@@ -36,7 +83,8 @@
    HBASE-1949  KeyValue expiration by Time-to-Live during major compaction is
                broken (Gary Helmling via Stack)
    HBASE-1957  Get-s can't set a Filter (Roman Kalyakin via Stack)
-   HBASE-1959  Compress tables during 0.19 to 0.20 migration (Dave Latham via Stack)
+   HBASE-1959  Compress tables during 0.19 to 0.20 migration (Dave Latham via
+               Stack)
    HBASE-1928  ROOT and META tables stay in transition state (making the system
                not usable) if the designated regionServer dies before the
                assignment is complete (Yannis Pavlidis via Stack)
@@ -47,6 +95,7 @@
                sometimes - Temporary fix
    HBASE-1965  On restart of branch, master complains about not being able
                to set safe mode
+   HBASE-2003  [shell] deleteall ignores column if specified
 
   IMPROVEMENTS
    HBASE-1899  Use scanner caching in shell count

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/HBase.rb
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/HBase.rb?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/HBase.rb (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/HBase.rb Thu Dec 17 20:26:29 2009
@@ -361,7 +361,8 @@
     def delete(row, column, timestamp = HConstants::LATEST_TIMESTAMP)
       now = Time.now 
       d = Delete.new(row.to_java_bytes, timestamp, nil)
-      d.deleteColumn(Bytes.toBytes(column))
+      split = KeyValue.parseColumn(column.to_java_bytes)
+      d.deleteColumn(split[0], split.length > 1 ? split[1] : nil, timestamp)
       @table.delete(d)
       @formatter.header()
       @formatter.footer(now)
@@ -370,6 +371,10 @@
     def deleteall(row, column = nil, timestamp = HConstants::LATEST_TIMESTAMP)
       now = Time.now 
       d = Delete.new(row.to_java_bytes, timestamp, nil)
+      if column != nil
+        split = KeyValue.parseColumn(column.to_java_bytes)
+        d.deleteColumns(split[0], split.length > 1 ? split[1] : nil, timestamp)
+      end
       @table.delete(d)
       @formatter.header()
       @formatter.footer(now)
@@ -411,12 +416,7 @@
           scan = Scan.new(startrow.to_java_bytes)
         end
         for c in columns
-          split = KeyValue.parseColumn(c.to_java_bytes)
-          if split[1] != nil
-            scan.addColumn(split[0], split[1])
-          else
-            scan.addFamily(split[0])
-          end
+          scan.addColumns(c)
         end
         if filter != nil
           scan.setFilter(filter)
@@ -435,7 +435,6 @@
       while i.hasNext()
         r = i.next().getRowResult()
         row = String.from_java_bytes r.getRow()
-        count += 1
         if limit != -1 and count >= limit
           break
         end
@@ -444,6 +443,7 @@
           cell = toString(column, v, maxlength)
           @formatter.row([row, "column=%s, %s" % [column, cell]])
         end
+        count += 1
       end
       @formatter.footer(now, count)
     end

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/hirb.rb
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/hirb.rb?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/hirb.rb (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/hirb.rb Thu Dec 17 20:26:29 2009
@@ -18,17 +18,11 @@
 # Some goodies for hirb. Should these be left up to the user's discretion?
 require 'irb/completion'
 
-# Hack to turn down zk logging so it don't spew over the shell
-# log4j.logger.org.apache.zookeeper=INFO
-logger = org.apache.log4j.Logger.getLogger("org.apache.zookeeper")
-logger.setLevel(org.apache.log4j.Level::WARN);
-
 # Add the $HBASE_HOME/bin directory, the location of this script, to the ruby
 # load path so I can load up my HBase ruby modules
 $LOAD_PATH.unshift File.dirname($PROGRAM_NAME)
-# Require formatter and hbase
+# Require formatter
 require 'Formatter'
-require 'HBase'
 
 # See if there are args for this shell. If any, read and then strip from ARGV
 # so they don't go through to irb.  Output shell 'usage' if user types '--help'
@@ -36,11 +30,13 @@
 HBase Shell command-line options:
  format        Formatter for outputting results: console | html. Default: console
  format-width  Width of table outputs. Default: 110 characters.
+ -d | --debug  Set DEBUG log levels.
 HERE
 found = []
 format = 'console'
 format_width = 110
 script2run = nil
+logLevel = org.apache.log4j.Level::ERROR  
 for arg in ARGV
   if arg =~ /^--format=(.+)/i
     format = $1
@@ -58,6 +54,10 @@
   elsif arg == '-h' || arg == '--help'
     puts cmdline_help
     exit
+  elsif arg == '-d' || arg == '--debug'
+    logLevel = org.apache.log4j.Level::DEBUG
+    $fullBackTrace = true
+    puts "Setting DEBUG log level..."  
   else
     # Presume it a script. Save it off for running later below
     # after we've set up some environment.
@@ -70,11 +70,20 @@
 for arg in found
   ARGV.delete(arg)
 end
+
 # Presume console format.
 # Formatter takes an :output_stream parameter, if you don't want STDOUT.
 @formatter = Formatter::Console.new(:format_width => format_width)
 # TODO, etc.  @formatter = Formatter::XHTML.new(STDOUT)
 
+# Set logging level to avoid verboseness
+logger = org.apache.log4j.Logger.getLogger("org.apache.zookeeper")
+logger.setLevel(logLevel);
+logger = org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase")
+logger.setLevel(logLevel);
+# Require HBase now after setting log levels
+require 'HBase'
+
 # Setup the HBase module.  Create a configuration.
 # Turn off retries in hbase and ipc.  Human doesn't want to wait on N retries.
 @configuration = org.apache.hadoop.hbase.HBaseConfiguration.new()
@@ -478,6 +487,7 @@
     IRB.setup(ap_path)
     @CONF[:IRB_NAME] = 'hbase'
     @CONF[:AP_NAME] = 'hbase'
+    @CONF[:BACK_TRACE_LIMIT] = 0 unless $fullBackTrace
     
     if @CONF[:SCRIPT]
       hirb = HIRB.new(nil, @CONF[:SCRIPT])

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/loadtable.rb
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/loadtable.rb?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/loadtable.rb (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/loadtable.rb Thu Dec 17 20:26:29 2009
@@ -108,7 +108,7 @@
 htd.addFamily(hcd)
 previouslastkey = HConstants::EMPTY_START_ROW
 count = map.size()
-for i in map.descendingIterator()
+for i in map.descendingKeySet().iterator()
   tuple = map.get(i)
   startkey = i
   count = count - 1

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/conf/hbase-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/conf/hbase-default.xml?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/conf/hbase-default.xml (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/conf/hbase-default.xml Thu Dec 17 20:26:29 2009
@@ -122,7 +122,7 @@
   </property>
   <property>
     <name>hbase.client.pause</name>
-    <value>2000</value>
+    <value>1000</value>
     <description>General client pause value.  Used mostly as value to wait
     before running a retry of a failed get, region lookup, etc.</description>
   </property>
@@ -161,16 +161,17 @@
   </property>
   <property>
     <name>hbase.regionserver.msginterval</name>
-    <value>3000</value>
+    <value>1000</value>
     <description>Interval between messages from the RegionServer to HMaster
-    in milliseconds.  Default is 3 seconds.
+    in milliseconds. Use a high value like 3000 for clusters with more than 10
+    nodes. Default is 1 second so that HBase seems more 'live'.
     </description>
   </property>
   <property>
     <name>hbase.regionserver.flushlogentries</name>
     <value>100</value>
     <description>Sync the HLog to the HDFS when it has accumulated this many
-    entries. Default 100. Value is checked on every HLog.append
+    entries. Default 100. Value is checked on every HLog.sync
     </description>
   </property>
   <property>
@@ -394,10 +395,15 @@
   <property>
     <name>zookeeper.session.timeout</name>
     <value>60000</value>
-    <description>ZooKeeper session timeout. This option is not used by HBase
-      directly, it is for the internals of ZooKeeper. HBase merely passes it in
-      whenever a connection is established to ZooKeeper. It is used by ZooKeeper
-      for hearbeats. In milliseconds.
+    <description>ZooKeeper session timeout.
+      HBase passes this to the zk quorum as suggested maximum time for a
+      session.  See http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions
+      "The client sends a requested timeout, the server responds with the
+      timeout that it can give the client. The current implementation
+      requires that the timeout be a minimum of 2 times the tickTime
+      (as set in the server configuration) and a maximum of 20 times
+      the tickTime." Set the zk ticktime with hbase.zookeeper.property.tickTime.
+      In milliseconds.
     </description>
   </property>
   <property>
@@ -491,9 +497,10 @@
   -->
   <property>
     <name>hbase.zookeeper.property.tickTime</name>
-    <value>2000</value>
+    <value>3000</value>
     <description>Property from ZooKeeper's config zoo.cfg.
-    The number of milliseconds of each tick.
+    The number of milliseconds of each tick.  See
+    zookeeper.session.timeout description.
     </description>
   </property>
   <property>

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/README.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/README.txt?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/README.txt (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/README.txt Thu Dec 17 20:26:29 2009
@@ -42,7 +42,9 @@
    with 'cert' and end with '.pem'.
 
    Make sure the private part of your AWS SSH keypair exists in the same
-   directory as EC2_PRIVATE_KEY with the name id_rsa_root.
+   directory as EC2_PRIVATE_KEY with the name id_rsa_root. Also, insure that
+   the permissions on the private key file are 600 (ONLY owner readable/
+   writable).
 
 4) ./bin/hbase-ec2 launch-cluster <name> <nr-zoos> <nr-slaves>, e.g
 

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/create-hbase-image
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/create-hbase-image?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/create-hbase-image (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/create-hbase-image Thu Dec 17 20:26:29 2009
@@ -64,6 +64,10 @@
 
 # Copy setup scripts
 scp $SSH_OPTS "$bin"/hbase-ec2-env.sh "root@$HOSTNAME:/mnt"
+scp $SSH_OPTS "$bin"/functions.sh "root@$HOSTNAME:/mnt"
+if [ -f "$bin"/credentials.sh ] ; then
+  scp $SSH_OPTS "$bin"/credentials.sh "root@$HOSTNAME:/mnt"
+fi
 scp $SSH_OPTS "$bin"/image/create-hbase-image-remote "root@$HOSTNAME:/mnt"
 scp $SSH_OPTS "$bin"/image/ec2-run-user-data "root@$HOSTNAME:/etc/init.d"
 

Added: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/functions.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/functions.sh?rev=891871&view=auto
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/functions.sh (added)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/functions.sh Thu Dec 17 20:26:29 2009
@@ -0,0 +1,19 @@
+function getCredentialSetting {
+  name=$1
+  eval "val=\$$name"
+  if [ -z "$val" ] ; then
+    if [ -f "$bin"/credentials.sh ] ; then
+      val=`cat "$bin"/credentials.sh | grep $name | awk 'BEGIN { FS="=" } { print $2; }'`
+      if [ -z "$val" ] ; then
+        echo -n "$name: "
+        read -e val
+        echo "$name=$val" >> "$bin"/credentials.sh
+      fi
+    else
+      echo -n "$name: "
+      read -e val
+      echo "$name=$val" >> "$bin"/credentials.sh
+    fi
+    eval "$name=$val"
+  fi
+}

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/hbase-ec2-env.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/hbase-ec2-env.sh?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/hbase-ec2-env.sh (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/hbase-ec2-env.sh Thu Dec 17 20:26:29 2009
@@ -1,4 +1,4 @@
-# Set environment variables for running Hbase on Amazon EC2 here. All are required.
+# Set environment variables for running Hbase on Amazon EC2 here.
 
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -30,40 +30,24 @@
 # Your AWS certificate file -- must begin with 'cert' and end with '.pem'
 EC2_CERT=
 
-# Location of EC2 keys.
-# The default setting is probably OK if you set up EC2 following the Amazon Getting Started guide.
-EC2_KEYDIR=`dirname "$EC2_PRIVATE_KEY"`
-
-# Where your EC2 private key is stored (created when following the Amazon Getting Started guide).
-# You need to change this if you don't store this with your other EC2 keys.
-PRIVATE_KEY_PATH=`echo "$EC2_KEYDIR"/"id_rsa_root"`
-
-# SSH options used when connecting to EC2 instances.
-SSH_OPTS=`echo -q -i "$PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o ServerAliveInterval=30`
-
-# Global tool options
-TOOL_OPTS=`echo -K "$EC2_PRIVATE_KEY" -C "$EC2_CERT"`
+# Your AWS root SSH key
+EC2_ROOT_SSH_KEY=
 
 # The version of HBase to use.
-HBASE_VERSION=0.20.2-0.18.3
+HBASE_VERSION=0.20.2
 
 # The version of Hadoop to use.
-# HADOOP_VERSION=$HBASE_VERSION
-HADOOP_VERSION=0.18.3
+HADOOP_VERSION=0.20.1
 
 # The Amazon S3 bucket where the HBase AMI is stored.
-# The default value is for public images, so can be left if you are using running a public image.
 # Change this value only if you are creating your own (private) AMI
 # so you can store it in a bucket you own.
-S3_BUCKET=hbase-images
+#S3_BUCKET=hbase-images
+S3_BUCKET=iridiant-bundles
 
 # Enable public access web interfaces
-# XXX -- Generally, you do not want to do this
 ENABLE_WEB_PORTS=false
 
-# The script to run on instance boot.
-USER_DATA_FILE=hbase-ec2-init-remote.sh
-
 # Use only c1.xlarge unless you know what you are doing
 MASTER_INSTANCE_TYPE=${MASTER_INSTANCE_TYPE:-c1.xlarge}
 
@@ -73,6 +57,29 @@
 # Use only c1.medium unless you know what you are doing
 ZOO_INSTANCE_TYPE=${ZOO_INSTANCE_TYPE:-c1.medium}
 
+############################################################################
+
+# Generally, users do not need to edit below
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+. "$bin"/functions.sh
+getCredentialSetting 'AWS_ACCOUNT_ID'
+getCredentialSetting 'AWS_ACCESS_KEY_ID'
+getCredentialSetting 'AWS_SECRET_ACCESS_KEY'
+getCredentialSetting 'EC2_PRIVATE_KEY'
+getCredentialSetting 'EC2_CERT'
+getCredentialSetting 'EC2_ROOT_SSH_KEY'
+
+# SSH options used when connecting to EC2 instances.
+SSH_OPTS=`echo -q -i "$EC2_ROOT_SSH_KEY" -o StrictHostKeyChecking=no -o ServerAliveInterval=30`
+
+# EC2 command request timeout (seconds)
+REQUEST_TIMEOUT=300    # 5 minutes
+
+# Global tool options
+TOOL_OPTS=`echo -K "$EC2_PRIVATE_KEY" -C "$EC2_CERT" --request-timeout $REQUEST_TIMEOUT`
+
 # The EC2 group master name. CLUSTER is set by calling scripts
 CLUSTER_MASTER=$CLUSTER-master
 
@@ -85,6 +92,9 @@
 CLUSTER_ZOOKEEPER=$CLUSTER-zookeeper
 ZOOKEEPER_QUORUM_PATH=~/.hbase-quorum-$CLUSTER_ZOOKEEPER
 
+# The script to run on instance boot.
+USER_DATA_FILE=hbase-ec2-init-remote.sh
+
 # The version number of the installed JDK.
 JAVA_VERSION=1.6.0_17
 

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/hbase-ec2-init-remote.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/hbase-ec2-init-remote.sh?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/hbase-ec2-init-remote.sh (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/hbase-ec2-init-remote.sh Thu Dec 17 20:26:29 2009
@@ -13,13 +13,15 @@
  MASTER_HOST=`wget -q -O - http://169.254.169.254/latest/meta-data/local-hostname`
 fi
 HADOOP_HOME=`ls -d /usr/local/hadoop-*`
+HADOOP_VERSION=`echo $HADOOP_HOME | cut -d '-' -f 2`
 HBASE_HOME=`ls -d /usr/local/hbase-*`
+HBASE_VERSION=`echo $HBASE_HOME | cut -d '-' -f 2`
 
 ###############################################################################
 # Hadoop configuration
 ###############################################################################
 
-cat > $HADOOP_HOME/conf/hadoop-site.xml <<EOF
+cat > $HADOOP_HOME/conf/core-site.xml <<EOF
 <?xml version="1.0"?>
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <configuration>
@@ -31,6 +33,24 @@
   <name>fs.default.name</name>
   <value>hdfs://$MASTER_HOST:8020</value>
 </property>
+</configuration>
+EOF
+
+cat > $HADOOP_HOME/conf/hdfs-site.xml <<EOF
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+<property>
+  <name>fs.default.name</name>
+  <value>hdfs://$MASTER_HOST:8020</value>
+</property>
+</configuration>
+EOF
+
+cat > $HADOOP_HOME/conf/mapred-site.xml <<EOF
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
 <property>
   <name>mapred.job.tracker</name>
   <value>$MASTER_HOST:8021</value>

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/image/create-hbase-image-remote
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/image/create-hbase-image-remote?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/image/create-hbase-image-remote (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/image/create-hbase-image-remote Thu Dec 17 20:26:29 2009
@@ -30,8 +30,9 @@
 echo "Remote: INSTANCE_TYPE is $type"
 echo "Remote: ARCH is $arch"
 
-# Remove environment script since it contains sensitive information
+# Remove sensitive information
 rm -f "$bin"/hbase-ec2-env.sh
+rm -f "$bin"/credentials.sh
 
 # Install Java
 echo "Downloading and installing java binary."
@@ -98,7 +99,7 @@
 cd /usr/local/hbase-${HBASE_VERSION}
 wget -nv http://iridiant.s3.amazonaws.com/hbase/lzo-linux-${HADOOP_VERSION}.tar.gz
 tar xzf lzo-linux-${HADOOP_VERSION}.tar.gz
-rm -f lzo-linux-${HADOOP_VERSION}.tar.gz
+rm lzo-linux-${HADOOP_VERSION}.tar.gz
 
 # Bundle and upload image
 cd ~root

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-master
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-master?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-master (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-master Thu Dec 17 20:26:29 2009
@@ -88,7 +88,7 @@
   sleep 5
 done
 
-scp $SSH_OPTS $PRIVATE_KEY_PATH "root@$MASTER_EC2_HOST:/root/.ssh/id_rsa"
+scp $SSH_OPTS $EC2_ROOT_SSH_KEY "root@$MASTER_EC2_HOST:/root/.ssh/id_rsa"
 ssh $SSH_OPTS "root@$MASTER_EC2_HOST" "chmod 600 /root/.ssh/id_rsa"
 
 MASTER_IP=`dig +short $MASTER_EC2_HOST`

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-zookeeper
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-zookeeper?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-zookeeper (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/contrib/ec2/bin/launch-hbase-zookeeper Thu Dec 17 20:26:29 2009
@@ -49,7 +49,8 @@
 
 peers=""
 public_names=""
-for inst in `seq 1 $NO_INSTANCES` ; do 
+inst=0
+while [ $inst -lt $NO_INSTANCES ] ; do
   echo "Starting an AMI with ID $ZOO_AMI_IMAGE (arch $arch) in group $CLUSTER_ZOOKEEPER"
   INSTANCE=`ec2-run-instances $ZOO_AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_ZOOKEEPER -k root -t $type | grep INSTANCE | awk '{print $2}'`
   echo -n "Waiting for instance $INSTANCE to start: "
@@ -67,6 +68,7 @@
   public=`ec2-describe-instances $TOOL_OPTS $INSTANCE | grep running | awk '{print $4}'`
   echo "    Public DNS name is $public."
   public_names="$public_names $public"
+  inst=$(($inst + 1))
 done
 
 ZOOKEEPER_QUORUM=`echo $peers | sed -e 's/ /,/g'`

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HBaseConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HBaseConfiguration.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HBaseConfiguration.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HBaseConfiguration.java Thu Dec 17 20:26:29 2009
@@ -52,7 +52,7 @@
   
   /**
    * Returns the hash code value for this HBaseConfiguration. The hash code of a
-   * HBaseConfiguration is defined by the sum of the hash codes of its entries.
+   * HBaseConfiguration is defined by the xor of the hash codes of its entries.
    * 
    * @see Configuration#iterator() How the entries are obtained.
    */
@@ -66,5 +66,29 @@
     }
     return hash;
   }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (!(obj instanceof HBaseConfiguration))
+      return false;
+    
+    HBaseConfiguration otherConf = (HBaseConfiguration) obj;
+    Iterator<Entry<String, String>> propertyIterator = this.iterator();
+    while (propertyIterator.hasNext()) {
+      Entry<String, String> entry = propertyIterator.next();
+      String key = entry.getKey();
+      String value = entry.getValue();
+      if (!value.equals(otherConf.getRaw(key))) {
+        return false;
+      }
+    }
+    
+    return true;
+  }
+  
   
 }

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java Thu Dec 17 20:26:29 2009
@@ -24,10 +24,10 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeSet;
-import java.util.WeakHashMap;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.commons.logging.Log;
@@ -67,7 +67,17 @@
 public class HConnectionManager implements HConstants {
   private static final Delete [] DELETE_ARRAY_TYPE = new Delete[0];
   private static final Put [] PUT_ARRAY_TYPE = new Put[0];
-  
+
+  // Register a shutdown hook, one that cleans up RPC and closes zk sessions.
+  static {
+    Runtime.getRuntime().addShutdownHook(new Thread("HCM.shutdownHook") {
+      @Override
+      public void run() {
+        HConnectionManager.deleteAllConnections(true);
+      }
+    });
+  }
+
   /*
    * Not instantiable.
    */
@@ -75,12 +85,20 @@
     super();
   }
   
-  // A Map of master HBaseConfiguration -> connection information for that 
-  // instance. Note that although the Map is synchronized, the objects it 
-  // contains are mutable and hence require synchronized access to them
+  private static final int MAX_CACHED_HBASE_INSTANCES=31;
+  // A LRU Map of master HBaseConfiguration -> connection information for that 
+  // instance. The objects it contains are mutable and hence require
+  // synchronized access to them.  We set instances to 31.  The zk default max
+  // connections is 30 so should run into zk issues before hit this value of 31.
   private static 
   final Map<HBaseConfiguration, TableServers> HBASE_INSTANCES =
-    new WeakHashMap<HBaseConfiguration, TableServers>();
+    new LinkedHashMap<HBaseConfiguration, TableServers>
+      ((int) (MAX_CACHED_HBASE_INSTANCES/0.75F)+1, 0.75F, true) {
+      @Override
+      protected boolean removeEldestEntry(Map.Entry<HBaseConfiguration, TableServers> eldest) {
+        return size() > MAX_CACHED_HBASE_INSTANCES;
+      }
+  };
   
   private static final Map<String, ClientZKWatcher> ZK_WRAPPERS = 
     new HashMap<String, ClientZKWatcher>();
@@ -186,8 +204,10 @@
      */
     public void process(WatchedEvent event) {
       KeeperState state = event.getState();
-      LOG.debug("Got ZooKeeper event, state: " + state + ", type: "
-          + event.getType() + ", path: " + event.getPath());
+      if(!state.equals(KeeperState.SyncConnected)) {
+        LOG.debug("Got ZooKeeper event, state: " + state + ", type: "
+            + event.getType() + ", path: " + event.getPath());
+      }
       if (state == KeeperState.Expired) {
         resetZooKeeper();
       }
@@ -268,7 +288,7 @@
             "Unable to find region server interface " + serverClassName, e);
       }
 
-      this.pause = conf.getLong("hbase.client.pause", 2 * 1000);
+      this.pause = conf.getLong("hbase.client.pause", 1 * 1000);
       this.numRetries = conf.getInt("hbase.client.retries.number", 10);
       this.maxRPCAttempts = conf.getInt("hbase.client.rpc.maxattempts", 1);
       this.rpcTimeout = conf.getLong("hbase.regionserver.lease.period", 60000);
@@ -689,7 +709,7 @@
             if (LOG.isDebugEnabled()) {
               LOG.debug("locateRegionInMeta attempt " + tries + " of " +
                 this.numRetries + " failed; retrying after sleep of " +
-                getPauseTime(tries), e);
+                getPauseTime(tries) + " because: " + e.getMessage());
             }
             relocateRegion(parentTable, metaKey);
           } else {
@@ -846,7 +866,9 @@
       SoftValueSortedMap<byte [], HRegionLocation> tableLocations =
         getTableLocations(tableName);
       if (tableLocations.put(startKey, location) == null) {
-        LOG.debug("Cached location " + location);
+        LOG.debug("Cached location for " +
+            location.getRegionInfo().getRegionNameAsString() +
+            " is " + location.getServerAddress());
       }
     }
     
@@ -1241,4 +1263,4 @@
       }
     }
   } 
-}
\ No newline at end of file
+}

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java Thu Dec 17 20:26:29 2009
@@ -132,12 +132,8 @@
    * @param value column value
    */
   public Put add(byte [] family, byte [] qualifier, long ts, byte [] value) {
-    List<KeyValue> list = familyMap.get(family);
-    if(list == null) {
-      list = new ArrayList<KeyValue>(0);
-    }
-    KeyValue kv = new KeyValue(this.row, family, qualifier, ts, 
-      KeyValue.Type.Put, value); 
+    List<KeyValue> list = getKeyValueList(family);
+    KeyValue kv = createPutKeyValue(family, qualifier, ts, value);
     list.add(kv);
     familyMap.put(kv.getFamily(), list);
     return this;
@@ -151,10 +147,7 @@
    */
   public Put add(KeyValue kv) throws IOException{
     byte [] family = kv.getFamily();
-    List<KeyValue> list = familyMap.get(family);
-    if(list == null) {
-      list = new ArrayList<KeyValue>();
-    }
+    List<KeyValue> list = getKeyValueList(family);
     //Checking that the row of the kv is the same as the put
     int res = Bytes.compareTo(this.row, 0, row.length, 
         kv.getBuffer(), kv.getRowOffset(), kv.getRowLength());
@@ -168,7 +161,162 @@
     familyMap.put(family, list);
     return this;
   }
+
+  /**
+   * Create a KeyValue with this objects row key and the Put identifier.
+   * 
+   * @param family
+   * @param qualifier
+   * @param ts
+   * @param value
+   * @return a KeyValue with this objects row key and the Put identifier.
+   */
+  private KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts,
+      byte[] value) {
+  return  new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, 
+      value);
+  }
+  
+  /**
+   * A convenience method to determine if this object's familyMap contains 
+   * a value assigned to the given family & qualifier.
+   * Both given arguments must match the KeyValue object to return true.
+   * 
+   * @param family
+   * @param qualifier
+   * @return returns true if the given family and qualifier already has an
+   * existing KeyValue object in the family map.
+   */
+  public boolean has(byte [] family, byte [] qualifier) {
+  return has(family, qualifier, this.timestamp, new byte[0], true, true);
+  }
+  
+  /**
+   * A convenience method to determine if this object's familyMap contains 
+   * a value assigned to the given family, qualifier and timestamp.
+   * All 3 given arguments must match the KeyValue object to return true.
+   * 
+   * @param family
+   * @param qualifier
+   * @param ts
+   * @return returns true if the given family, qualifier and timestamp already has an
+   * existing KeyValue object in the family map.
+   */
+  public boolean has(byte [] family, byte [] qualifier, long ts) {
+  return has(family, qualifier, ts, new byte[0], false, true);
+  }
+  
+  /**
+   * A convenience method to determine if this object's familyMap contains 
+   * a value assigned to the given family, qualifier and timestamp.
+   * All 3 given arguments must match the KeyValue object to return true.
+   * 
+   * @param family
+   * @param qualifier
+   * @param value
+   * @return returns true if the given family, qualifier and value already has an
+   * existing KeyValue object in the family map.
+   */
+  public boolean has(byte [] family, byte [] qualifier, byte [] value) {
+    return has(family, qualifier, this.timestamp, value, true, false);
+  }
+  
+  /**
+   * A convenience method to determine if this object's familyMap contains 
+   * the given value assigned to the given family, qualifier and timestamp.
+   * All 4 given arguments must match the KeyValue object to return true.
+   * 
+   * @param family
+   * @param qualifier
+   * @param ts
+   * @param value
+   * @return returns true if the given family, qualifier timestamp and value 
+   * already has an existing KeyValue object in the family map.
+   */
+  public boolean has(byte [] family, byte [] qualifier, long ts, byte [] value) {
+      return has(family, qualifier, ts, value, false, false);
+  }
+  
+  /**
+   * Private method to determine if this object's familyMap contains 
+   * the given value assigned to the given family, qualifier and timestamp
+   * respecting the 2 boolean arguments
+   * 
+   * @param family
+   * @param qualifier
+   * @param ts
+   * @param value
+   * @param ignoreTS
+   * @param ignoreValue
+   * @return returns true if the given family, qualifier timestamp and value 
+   * already has an existing KeyValue object in the family map.
+   */
+  private boolean has(byte [] family, byte [] qualifier, long ts, byte [] value, 
+      boolean ignoreTS, boolean ignoreValue) {
+    List<KeyValue> list = getKeyValueList(family);
+    if (list.size() == 0 ) {
+      return false;
+    }
+    if (!ignoreTS && !ignoreValue) {
+      KeyValue kv = createPutKeyValue(family, qualifier, ts, value);
+      return (list.contains(kv));
+    } else if (ignoreValue) {
+      for (KeyValue kv: list) {
+        if (Arrays.equals(kv.getFamily(), family) && Arrays.equals(kv.getQualifier(), qualifier)
+            && kv.getTimestamp() == ts) {
+          return true;
+        }
+      }
+    } else if (ignoreTS) {
+      for (KeyValue kv: list) {
+      if (Arrays.equals(kv.getFamily(), family) && Arrays.equals(kv.getQualifier(), qualifier)
+              && Arrays.equals(kv.getValue(), value)) {
+          return true;
+        }
+      }
+    } else {
+      for (KeyValue kv: list) {
+      if (Arrays.equals(kv.getFamily(), family) && Arrays.equals(
+          kv.getQualifier(), qualifier)) {
+          return true;
+        }
+    }
+    }
+    return false;
+  }
   
+  /**
+   * Returns a list of all KeyValue objects with matching column family and qualifier.
+   * 
+   * @param family
+   * @param qualifier
+   * @return a list of KeyValue objects with the matching family and qualifier, 
+   * returns an empty list if one doesnt exist for the given family.
+   */
+  public List<KeyValue> get(byte[] family, byte[] qualifier) {
+    List<KeyValue> filteredList = new ArrayList<KeyValue>();
+    for (KeyValue kv: getKeyValueList(family)) {
+      if (Arrays.equals(kv.getQualifier(), qualifier)) {
+        filteredList.add(kv);
+      }
+    }
+    return filteredList;
+  }
+
+  /**
+   * Creates an empty list if one doesnt exist for the given column family
+   * or else it returns the associated list of KeyValue objects.
+   * 
+   * @param family
+   * @return a list of KeyValue objects, returns an empty list if one doesnt exist.
+   */
+  private List<KeyValue> getKeyValueList(byte[] family) {
+    List<KeyValue> list = familyMap.get(family);
+    if(list == null) {
+      list = new ArrayList<KeyValue>(0);
+    }
+    return list;
+  }
   
   /**
    * Method for retrieving the put's familyMap

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Scan.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Scan.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Scan.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Scan.java Thu Dec 17 20:26:29 2009
@@ -265,10 +265,14 @@
   
   /**
    * Get versions of columns only within the specified timestamp range,
-   * [minStamp, maxStamp).
+   * [minStamp, maxStamp).  Note, default maximum versions to return is 1.  If
+   * your time range spans more than one version and you want all versions
+   * returned, up the number of versions beyond the defaut.
    * @param minStamp minimum timestamp value, inclusive
    * @param maxStamp maximum timestamp value, exclusive
    * @throws IOException if invalid time range
+   * @see {@link #setMaxVersions()}
+   * @see {@link #setMaxVersions(int)}
    */
   public Scan setTimeRange(long minStamp, long maxStamp)
   throws IOException {
@@ -277,8 +281,13 @@
   }
   
   /**
-   * Get versions of columns with the specified timestamp.
-   * @param timestamp version timestamp  
+   * Get versions of columns with the specified timestamp. Note, default maximum
+   * versions to return is 1.  If your time range spans more than one version
+   * and you want all versions returned, up the number of versions beyond the
+   * defaut.
+   * @param timestamp version timestamp
+   * @see {@link #setMaxVersions()}
+   * @see {@link #setMaxVersions(int)}
    */
   public Scan setTimeStamp(long timestamp) {
     try {

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java Thu Dec 17 20:26:29 2009
@@ -29,7 +29,6 @@
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.ipc.RemoteException;
 
-
 /**
  * Retries scanner operations such as create, next, etc.
  * Used by {@link ResultScanner}s made by {@link HTable}.

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java Thu Dec 17 20:26:29 2009
@@ -27,7 +27,7 @@
 </ul>
 
  <h2><a name="overview">Overview</a></h2>
- <p>To administer HBase, create and drop tables, list and alter tables, 
+ <p>To administer HBase, create and drop tables, list and alter tables,
  use {@link org.apache.hadoop.hbase.client.HBaseAdmin}.  Once created, table access is via an instance
  of {@link org.apache.hadoop.hbase.client.HTable}.  You add content to a table a row at a time.  To insert,
  create an instance of a {@link org.apache.hadoop.hbase.client.Put} object.  Specify value, target column
@@ -53,11 +53,11 @@
  This means that the ZooKeeper quorum to use must be on the client CLASSPATH.
  Usually this means make sure the client can find your <code>hbase-site.xml</code>.
  </p>
- 
+
 <h2><a name="client_example">Example API Usage</a></h2>
 
-<p>Once you have a running HBase, you probably want a way to hook your application up to it. 
-  If your application is in Java, then you should use the Java API. Here's an example of what 
+<p>Once you have a running HBase, you probably want a way to hook your application up to it.
+  If your application is in Java, then you should use the Java API. Here's an example of what
   a simple client might look like.  This example assumes that you've created a table called
   "myTable" with a column family called "myColumnFamily".
 </p>
@@ -87,7 +87,7 @@
     HBaseConfiguration config = new HBaseConfiguration();
 
     // This instantiates an HTable object that connects you to
-    // the "myLittleHBaseTable" table. 
+    // the "myLittleHBaseTable" table.
     HTable table = new HTable(config, "myLittleHBaseTable");
 
     // To add to a row, use Put.  A Put constructor takes the name of the row
@@ -100,11 +100,11 @@
     // applies current time to the edits.
     Put p = new Put(Bytes.toBytes("myLittleRow"));
 
-    // To set the value you'd like to update in the row 'myRow', specify the
-    // column family, column qualifier, and value of the table cell you'd like
-    // to update.  The column family must already exist in your table schema.
-    // The qualifier can be anything.  All must be specified as byte arrays as
-    // hbase is all about byte arrays.  Lets pretend the table
+    // To set the value you'd like to update in the row 'myLittleRow', specify
+    // the column family, column qualifier, and value of the table cell you'd
+    // like to update.  The column family must already exist in your table
+    // schema.  The qualifier can be anything.  All must be specified as byte
+    // arrays as hbase is all about byte arrays.  Lets pretend the table
     // 'myLittleHBaseTable' was created with a family 'myLittleFamily'.
     p.add(Bytes.toBytes("myLittleFamily"), Bytes.toBytes("someQualifier"),
       Bytes.toBytes("Some Value"));
@@ -156,12 +156,12 @@
 </pre></blockquote>
 </div>
 
-<p>There are many other methods for putting data into and getting data out of 
+<p>There are many other methods for putting data into and getting data out of
   HBase, but these examples should get you started. See the HTable javadoc for
-  more methods. Additionally, there are methods for managing tables in the 
+  more methods. Additionally, there are methods for managing tables in the
   HBaseAdmin class.</p>
 
-<p>If your client is NOT Java, then you should consider the Thrift or REST 
+<p>If your client is NOT Java, then you should consider the Thrift or REST
   libraries.</p>
 
 <h2><a name="related" >Related Documentation</a></h2>
@@ -173,9 +173,9 @@
 </pre></code>
 </div>
 
-<p>There are many other methods for putting data into and getting data out of 
+<p>There are many other methods for putting data into and getting data out of
   HBase, but these examples should get you started. See the HTable javadoc for
-  more methods. Additionally, there are methods for managing tables in the 
+  more methods. Additionally, there are methods for managing tables in the
   HBaseAdmin class.</p>
 
 </body>

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java Thu Dec 17 20:26:29 2009
@@ -49,10 +49,13 @@
  * <p>
  * To prevent the entire row from being emitted if the column is not found
  * on a row, use {@link #setFilterIfMissing}.
- * <p>
  * Otherwise, if the column is found, the entire row will be emitted only if
  * the value passes.  If the value fails, the row will be filtered out.
  * <p>
+ * In order to test values of previous versions (timestamps), set
+ * {@link #setLatestVersionOnly} to false. The default is true, meaning that
+ * only the latest version's value is tested and all previous versions are ignored.
+ * <p>
  * To filter based on the value of all scanned columns, use {@link ValueFilter}.
  */
 public class SingleColumnValueFilter implements Filter {
@@ -65,7 +68,8 @@
   private boolean foundColumn = false;
   private boolean matchedColumn = false;
   private boolean filterIfMissing = false;
-  
+  private boolean latestVersionOnly = true;
+
   /**
    * Writable constructor, do not use.
    */
@@ -121,7 +125,7 @@
     if (this.matchedColumn) {
       // We already found and matched the single column, all keys now pass
       return ReturnCode.INCLUDE;
-    } else if (this.foundColumn) {
+    } else if (this.latestVersionOnly && this.foundColumn) {
       // We found but did not match the single column, skip to next row
       return ReturnCode.NEXT_ROW;
     }
@@ -131,7 +135,7 @@
     foundColumn = true;
     if (filterColumnValue(keyValue.getBuffer(),
         keyValue.getValueOffset(), keyValue.getValueLength())) {
-      return ReturnCode.NEXT_ROW;
+      return this.latestVersionOnly? ReturnCode.NEXT_ROW: ReturnCode.INCLUDE;
     }
     this.matchedColumn = true;
     return ReturnCode.INCLUDE;
@@ -197,6 +201,26 @@
     this.filterIfMissing = filterIfMissing;
   }
 
+  /**
+   * Get whether only the latest version of the column value should be compared.
+   * If true, the row will be returned if only the latest version of the column
+   * value matches. If false, the row will be returned if any version of the
+   * column value matches. The default is true.
+   */
+  public boolean getLatestVersionOnly() {
+    return latestVersionOnly;
+  }
+
+  /**
+   * Set whether only the latest version of the column value should be compared.
+   * If true, the row will be returned if only the latest version of the column
+   * value matches. If false, the row will be returned if any version of the
+   * column value matches. The default is true.
+   */
+  public void setLatestVersionOnly(boolean latestVersionOnly) {
+    this.latestVersionOnly = latestVersionOnly;
+  }
+
   public void readFields(final DataInput in) throws IOException {
     this.columnFamily = Bytes.readByteArray(in);
     if(this.columnFamily.length == 0) {
@@ -212,6 +236,7 @@
     this.foundColumn = in.readBoolean();
     this.matchedColumn = in.readBoolean();
     this.filterIfMissing = in.readBoolean();
+    this.latestVersionOnly = in.readBoolean();
   }
 
   public void write(final DataOutput out) throws IOException {
@@ -223,5 +248,6 @@
     out.writeBoolean(foundColumn);
     out.writeBoolean(matchedColumn);
     out.writeBoolean(filterIfMissing);
+    out.writeBoolean(latestVersionOnly);
   }
-}
\ No newline at end of file
+}

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java Thu Dec 17 20:26:29 2009
@@ -60,8 +60,6 @@
   
   private final HashSet<HRegion> regionsInQueue = new HashSet<HRegion>();
 
-  private volatile int limit = 1;
-
   /** @param server */
   public CompactSplitThread(HRegionServer server) {
     super();
@@ -81,18 +79,9 @@
         continue;
       }
     }
-    int count = 0;
     while (!this.server.isStopRequested()) {
       HRegion r = null;
       try {
-        if ((limit > 0) && (++count > limit)) {
-          try {
-            Thread.sleep(this.frequency);
-          } catch (InterruptedException ex) {
-            continue;
-          }
-          count = 0;
-        }
         r = compactionQueue.poll(this.frequency, TimeUnit.MILLISECONDS);
         if (r != null && !this.server.isStopRequested()) {
           synchronized (regionsInQueue) {
@@ -231,18 +220,6 @@
   }
 
   /**
-   * Sets the number of compactions allowed per cycle.
-   * @param limit the number of compactions allowed, or -1 to unlimit
-   */
-  void setLimit(int limit) {
-    this.limit = limit;
-  }
-
-  int getLimit() {
-    return this.limit;
-  }
-
-  /**
    * Only interrupt once it's done with a run through the work loop.
    */ 
   void interruptIfNecessary() {

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java Thu Dec 17 20:26:29 2009
@@ -379,7 +379,7 @@
       LOG.debug("Found " + sequenceNumbers.size() + " hlogs to remove " +
         " out of total " + this.outputfiles.size() + "; " +
         "oldest outstanding seqnum is " + oldestOutstandingSeqNum +
-        " from region " + Bytes.toString(oldestRegion));
+        " from region " + Bytes.toStringBinary(oldestRegion));
     }
     if (sequenceNumbers.size() > 0) {
       for (Long seq : sequenceNumbers) {
@@ -392,7 +392,7 @@
         oldestRegion: getOldestRegion(oldestOutstandingSeqNum);
       LOG.info("Too many hlogs: logs=" + countOfLogs + ", maxlogs=" +
         this.maxLogs + "; forcing flush of region with oldest edits: " +
-        Bytes.toString(regionToFlush));
+        Bytes.toStringBinary(regionToFlush));
     }
     return regionToFlush;
   }
@@ -898,7 +898,7 @@
                 LinkedList<HLogEntry> queue = logEntries.get(regionName);
                 if (queue == null) {
                   queue = new LinkedList<HLogEntry>();
-                  LOG.debug("Adding queue for " + Bytes.toString(regionName));
+                  LOG.debug("Adding queue for " + Bytes.toStringBinary(regionName));
                   logEntries.put(regionName, queue);
                 }
                 HLogEntry hle = new HLogEntry(val, key);
@@ -945,7 +945,7 @@
         ExecutorService threadPool =
           Executors.newFixedThreadPool(logWriterThreads);
         for (final byte[] key : logEntries.keySet()) {
-          Thread thread = new Thread(Bytes.toString(key)) {
+          Thread thread = new Thread(Bytes.toStringBinary(key)) {
             @Override
             public void run() {
               LinkedList<HLogEntry> entries = logEntries.get(key);
@@ -968,11 +968,18 @@
                     Path oldlogfile = null;
                     SequenceFile.Reader old = null;
                     if (fs.exists(logfile)) {
-                      LOG.warn("Old hlog file " + logfile
-                        + " already exists. Copying existing file to new file");
-                      oldlogfile = new Path(logfile.toString() + ".old");
-                      fs.rename(logfile, oldlogfile);
-                      old = new SequenceFile.Reader(fs, oldlogfile, conf);
+                      FileStatus stat = fs.getFileStatus(logfile);
+                      if (stat.getLen() <= 0) {
+                        LOG.warn("Old hlog file " + logfile + " is zero " +
+                          "length. Deleting existing file");
+                        fs.delete(logfile, false);
+                      } else {
+                        LOG.warn("Old hlog file " + logfile + " already " +
+                          "exists. Copying existing file to new file");
+                        oldlogfile = new Path(logfile.toString() + ".old");
+                        fs.rename(logfile, oldlogfile);
+                        old = new SequenceFile.Reader(fs, oldlogfile, conf);
+                      }
                     }
                     SequenceFile.Writer w =
                       SequenceFile.createWriter(fs, conf, logfile,
@@ -981,7 +988,7 @@
                     logWriters.put(key, wap);
                     if (LOG.isDebugEnabled()) {
                       LOG.debug("Creating new hlog file writer for path "
-                          + logfile + " and region " + Bytes.toString(key));
+                          + logfile + " and region " + Bytes.toStringBinary(key));
                     }
 
                     if (old != null) {
@@ -1004,12 +1011,12 @@
                 }
                 if (LOG.isDebugEnabled()) {
                   LOG.debug("Applied " + count + " total edits to "
-                      + Bytes.toString(key) + " in "
+                      + Bytes.toStringBinary(key) + " in "
                       + (System.currentTimeMillis() - threadTime) + "ms");
                 }
               } catch (IOException e) {
                 e = RemoteExceptionHandler.checkIOException(e);
-                LOG.warn("Got while writing region " + Bytes.toString(key)
+                LOG.warn("Got while writing region " + Bytes.toStringBinary(key)
                     + " log " + e);
                 e.printStackTrace();
               }

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Thu Dec 17 20:26:29 2009
@@ -207,9 +207,6 @@
   LogRoller hlogRoller;
   LogFlusher hlogFlusher;
   
-  // limit compactions while starting up
-  CompactionLimitThread compactionLimitThread;
-
   // flag set after we're done setting up server threads (used for testing)
   protected volatile boolean isOnline;
 
@@ -257,7 +254,7 @@
     // Config'ed params
     this.numRetries =  conf.getInt("hbase.client.retries.number", 2);
     this.threadWakeFrequency = conf.getInt(THREAD_WAKE_FREQUENCY, 10 * 1000);
-    this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 * 1000);
+    this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 1 * 1000);
     this.serverLeaseTimeout =
       conf.getInt("hbase.master.lease.period", 120 * 1000);
 
@@ -904,61 +901,6 @@
     return this.fsOk;
   }
 
-  /**
-   * Thread for toggling safemode after some configurable interval.
-   */
-  private class CompactionLimitThread extends Thread {
-    protected CompactionLimitThread() {}
-
-    @Override
-    public void run() {
-      // First wait until we exit safe mode
-      synchronized (safeMode) {
-        while(safeMode.get()) {
-          LOG.debug("Waiting to exit safe mode");
-          try {
-            safeMode.wait();
-          } catch (InterruptedException e) {
-            // ignore
-          }
-        }
-      }
-
-      // now that safemode is off, slowly increase the per-cycle compaction
-      // limit, finally setting it to unlimited (-1)
-
-      int compactionCheckInterval = 
-        conf.getInt("hbase.regionserver.thread.splitcompactcheckfrequency",
-            20 * 1000);
-      final int limitSteps[] = {
-        1, 1, 1, 1,
-        2, 2, 2, 2, 2, 2,
-        3, 3, 3, 3, 3, 3, 3, 3, 
-        4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
-        -1
-      };
-      for (int i = 0; i < limitSteps.length; i++) {
-        // Just log changes.
-        if (compactSplitThread.getLimit() != limitSteps[i] &&
-            LOG.isDebugEnabled()) {
-          LOG.debug("setting compaction limit to " + limitSteps[i]);
-        }
-        compactSplitThread.setLimit(limitSteps[i]);
-        try {
-          Thread.sleep(compactionCheckInterval);
-        } catch (InterruptedException ex) {
-          // unlimit compactions before exiting
-          compactSplitThread.setLimit(-1);
-          if (LOG.isDebugEnabled()) {
-            LOG.debug(this.getName() + " exiting on interrupt");
-          }
-          return;
-        }
-      }
-      LOG.info("compactions no longer limited");
-    }
-  }
-
   /*
    * Thread to shutdown the region server in an orderly manner.  This thread
    * is registered as a shutdown hook in the HRegionServer constructor and is
@@ -1222,17 +1164,6 @@
       } 
     }
 
-    // Set up the safe mode handler if safe mode has been configured.
-    if (!conf.getBoolean("hbase.regionserver.safemode", true)) {
-      safeMode.set(false);
-      compactSplitThread.setLimit(-1);
-      LOG.debug("skipping safe mode");
-    } else {
-      this.compactionLimitThread = new CompactionLimitThread();
-      Threads.setDaemonThreadRunning(this.compactionLimitThread, n + ".safeMode",
-        handler);
-    }
-
     // Start Server.  This service is like leases in that it internally runs
     // a thread.
     this.server.start();
@@ -1270,9 +1201,13 @@
     if (this.toDo.isEmpty()) {
       return;
     }
-    // This iterator is 'safe'.  We are guaranteed a view on state of the
-    // queue at time iterator was taken out.  Apparently goes from oldest.
+    // This iterator isn't safe if elements are gone and HRS.Worker could
+    // remove them (it already checks for null there). Goes from oldest.
     for (ToDoEntry e: this.toDo) {
+      if(e == null) {
+        LOG.warn("toDo gave a null entry during iteration");
+        break;
+      }
       HMsg msg = e.msg;
       if (msg != null) {
         if (msg.isType(HMsg.Type.MSG_REGION_OPEN)) {
@@ -1808,7 +1743,9 @@
     this.requestCount.incrementAndGet();
     HRegion region = getRegion(regionName);
     try {
-      cacheFlusher.reclaimMemStoreMemory();
+      if (!region.getRegionInfo().isMetaTable()) {
+        this.cacheFlusher.reclaimMemStoreMemory();
+      }
       region.put(put, getLockFromId(put.getLockId()));
     } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
@@ -1822,7 +1759,9 @@
     checkOpen();
     try {
       HRegion region = getRegion(regionName);
-      this.cacheFlusher.reclaimMemStoreMemory();
+      if (!region.getRegionInfo().isMetaTable()) {
+        this.cacheFlusher.reclaimMemStoreMemory();
+      }
       Integer[] locks = new Integer[puts.length];
       for (i = 0; i < puts.length; i++) {
         this.requestCount.incrementAndGet();
@@ -1863,7 +1802,9 @@
     this.requestCount.incrementAndGet();
     HRegion region = getRegion(regionName);
     try {
-      cacheFlusher.reclaimMemStoreMemory();
+      if (!region.getRegionInfo().isMetaTable()) {
+        this.cacheFlusher.reclaimMemStoreMemory();
+      }
       return region.checkAndPut(row, family, qualifier, value, put,
           getLockFromId(put.getLockId()), true);
     } catch (Throwable t) {
@@ -2015,10 +1956,12 @@
     checkOpen();
     try {
       boolean writeToWAL = true;
-      this.cacheFlusher.reclaimMemStoreMemory();
       this.requestCount.incrementAndGet();
-      Integer lid = getLockFromId(delete.getLockId());
       HRegion region = getRegion(regionName);
+      if (!region.getRegionInfo().isMetaTable()) {
+        this.cacheFlusher.reclaimMemStoreMemory();
+      }
+      Integer lid = getLockFromId(delete.getLockId());
       region.delete(delete, lid, writeToWAL);
     } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
@@ -2032,9 +1975,11 @@
     checkOpen();
     try {
       boolean writeToWAL = true;
-      this.cacheFlusher.reclaimMemStoreMemory();
-      Integer[] locks = new Integer[deletes.length];
       HRegion region = getRegion(regionName);
+      if (!region.getRegionInfo().isMetaTable()) {
+        this.cacheFlusher.reclaimMemStoreMemory();
+      }
+      Integer[] locks = new Integer[deletes.length];
       for (i = 0; i < deletes.length; i++) {
         this.requestCount.incrementAndGet();
         locks[i] = getLockFromId(deletes[i].getLockId());

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/MurmurHash.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/MurmurHash.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/MurmurHash.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/MurmurHash.java Thu Dec 17 20:26:29 2009
@@ -63,13 +63,13 @@
 
     if (left != 0) {
       if (left >= 3) {
-        h ^= data[length - 3] << 16;
+        h ^= data[len_m + 2] << 16;
       }
       if (left >= 2) {
-        h ^= data[length - 2] << 8;
+        h ^= data[len_m + 1] << 8;
       }
       if (left >= 1) {
-        h ^= data[length - 1];
+        h ^= data[len_m];
       }
 
       h *= m;

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java Thu Dec 17 20:26:29 2009
@@ -24,7 +24,12 @@
 import java.io.InputStream;
 import java.io.PrintWriter;
 import java.net.UnknownHostException;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
 import java.util.Properties;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Enumeration;
 import java.util.Map.Entry;
 
 import org.apache.commons.logging.Log;
@@ -90,22 +95,32 @@
     return address.equals("localhost") || address.equals("127.0.0.1");
   }
 
-  private static boolean hostEquals(String addrA, String addrB) {
-    if (addrA.contains(".") && addrB.contains(".")) {
-      return addrA.equals(addrB);
-    }
-    String hostA = StringUtils.simpleHostname(addrA);
-    String hostB = StringUtils.simpleHostname(addrB);
-    return hostA.equals(hostB);
-  }
+  private static void writeMyID(Properties properties) throws IOException {
+    long myId = -1;
 
-  private static void writeMyID(Properties properties) throws UnknownHostException, IOException {
     HBaseConfiguration conf = new HBaseConfiguration();
     String myAddress = DNS.getDefaultHost(
         conf.get("hbase.zookeeper.dns.interface","default"),
         conf.get("hbase.zookeeper.dns.nameserver","default"));
 
-    long myId = -1;
+    List<String> ips = new ArrayList<String>();
+
+    // Add what could be the best (configured) match
+    ips.add(myAddress.contains(".") ?
+        myAddress :
+        StringUtils.simpleHostname(myAddress));
+
+    // For all nics get all hostnames and IPs
+    Enumeration<?> nics = NetworkInterface.getNetworkInterfaces();
+    while(nics.hasMoreElements()) {
+      Enumeration<?> rawAdrs =
+          ((NetworkInterface)nics.nextElement()).getInetAddresses();
+      while(rawAdrs.hasMoreElements()) {
+        InetAddress inet = (InetAddress) rawAdrs.nextElement();
+        ips.add(StringUtils.simpleHostname(inet.getHostName()));
+        ips.add(inet.getHostAddress());
+      }
+    }
 
     for (Entry<Object, Object> entry : properties.entrySet()) {
       String key = entry.getKey().toString().trim();
@@ -115,9 +130,7 @@
         long id = Long.parseLong(key.substring(dot + 1));
         String[] parts = value.split(":");
         String address = parts[0];
-        if (addressIsLocalHost(address) || hostEquals(myAddress, address)) {
-          LOG.debug("found my address: " + myAddress + ", in list: " + address +
-                    ", setting myId to " + id);
+        if (addressIsLocalHost(address) || ips.contains(address)) {
           myId = id;
           break;
         }

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java Thu Dec 17 20:26:29 2009
@@ -19,8 +19,12 @@
  */
 package org.apache.hadoop.hbase.zookeeper;
 
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
 import java.net.InetAddress;
+import java.net.Socket;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.List;
@@ -176,8 +180,62 @@
     for (HServerAddress address : scanRSDirectory()) {
       sb.append("\n    - ").append(address);
     }
+    sb.append("\n  Quorum Server Statistics:");
+    String[] servers = quorumServers.split(",");
+    for (String server : servers) {
+      sb.append("\n    - ").append(server);
+      try {
+        String[] stat = getServerStats(server);
+        for (String s : stat) {
+          sb.append("\n        ").append(s);
+        }
+      } catch (Exception e) {
+        sb.append("\n        ERROR: ").append(e.getMessage());
+      }
+    }
     return sb.toString();
   }
+  
+  /**
+   * Gets the statistics from the given server. Uses a 1 minute timeout.
+   * 
+   * @param server  The server to get the statistics from.
+   * @return The array of response strings.
+   * @throws IOException When the socket communication fails.
+   */
+  public String[] getServerStats(String server) 
+  throws IOException {
+    return getServerStats(server, 1 * 60 * 1000);
+  }
+  
+  /**
+   * Gets the statistics from the given server.
+   * 
+   * @param server  The server to get the statistics from.
+   * @param timeout  The socket timeout to use.
+   * @return The array of response strings.
+   * @throws IOException When the socket communication fails.
+   */
+  public String[] getServerStats(String server, int timeout) 
+  throws IOException {
+    String[] sp = server.split(":");
+    Socket socket = new Socket(sp[0], 
+      sp.length > 1 ? Integer.parseInt(sp[1]) : 2181);
+    socket.setSoTimeout(timeout);
+    PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
+    BufferedReader in = new BufferedReader(new InputStreamReader(
+      socket.getInputStream()));
+    out.println("stat");
+    out.flush();
+    ArrayList<String> res = new ArrayList<String>();
+    while (true) {
+      String line = in.readLine();
+      if (line != null) res.add(line);
+      else break;
+    }
+    socket.close();
+    return res.toArray(new String[res.size()]);
+  }
 
   private boolean exists(String znode) {
     try {

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/overview.html
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/overview.html?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/overview.html (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/overview.html Thu Dec 17 20:26:29 2009
@@ -296,7 +296,8 @@
 
 <p>For more information about setting up a ZooKeeper cluster on your own, see
 the ZooKeeper <a href="http://hadoop.apache.org/zookeeper/docs/current/zookeeperStarted.html">Getting Started Guide</a>.
-HBase currently uses ZooKeeper version 3.2.0, so any cluster setup with a 3.x.x version of ZooKeeper should work.</p>
+HBase currently uses ZooKeeper version 3.2.0, so any cluster setup with a
+3.x.x version of ZooKeeper should work.</p>
 
 <p>Of note, if you have made <em>HDFS client configuration</em> on your Hadoop cluster, HBase will not
 see this configuration unless you do one of the following:</p>
@@ -341,10 +342,10 @@
 hbase&gt; create "mylittletable", "mylittlecolumnfamily"
 hbase&gt; # To see the schema for you just created "mylittletable" table and its single "mylittlecolumnfamily", type
 hbase&gt; describe "mylittletable"
-hbase&gt; # To add a row whose id is "x", to the column "mylittlecolumnfamily:x" with a value of 'x', do
-hbase&gt; put "mylittletable", "x"
+hbase&gt; # To add a row whose id is "myrow", to the column "mylittlecolumnfamily:x" with a value of 'v', do
+hbase&gt; put "mylittletable", "myrow", "mylittlecolumnfamily:x", "v"
 hbase&gt; # To get the cell just added, do
-hbase&gt; get "mylittletable", "x"
+hbase&gt; get "mylittletable", "myrow"
 hbase&gt; # To scan you new table, do
 hbase&gt; scan "mylittletable"
 </pre>

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java Thu Dec 17 20:26:29 2009
@@ -88,13 +88,13 @@
   private static final int ONE_GB = 1024 * 1024 * 1000;
   private static final int ROWS_PER_GB = ONE_GB / ROW_LENGTH;
   
-  static final byte [] FAMILY_NAME = Bytes.toBytes("info");
-  static final byte [] QUALIFIER_NAME = Bytes.toBytes("data");
+  public static final byte [] FAMILY_NAME = Bytes.toBytes("info");
+  public static final byte [] QUALIFIER_NAME = Bytes.toBytes("data");
   
   protected static final HTableDescriptor TABLE_DESCRIPTOR;
   static {
     TABLE_DESCRIPTOR = new HTableDescriptor("TestTable");
-    TABLE_DESCRIPTOR.addFamily(new HColumnDescriptor(CATALOG_FAMILY));
+    TABLE_DESCRIPTOR.addFamily(new HColumnDescriptor(FAMILY_NAME));
   }
   
   private static final String RANDOM_READ = "randomRead";
@@ -599,7 +599,7 @@
    * consumes about 30% of CPU time.
    * @return Generated random value to insert into a table cell.
    */
-  static byte[] generateValue(final Random r) {
+  public static byte[] generateValue(final Random r) {
     byte [] b = new byte [ROW_LENGTH];
     r.nextBytes(b);
     return b;

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java Thu Dec 17 20:26:29 2009
@@ -78,7 +78,7 @@
       new RegexStringComparator(QUICK_REGEX));
   }
 
-  private void basicFilterTests(Filter filter)
+  private void basicFilterTests(SingleColumnValueFilter filter)
       throws Exception {
     KeyValue kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_2);
     assertTrue("basicFilter1", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE);
@@ -94,6 +94,13 @@
     assertTrue("basicFilter4", filter.filterKeyValue(kv) == Filter.ReturnCode.NEXT_ROW);
     assertFalse("basicFilterAllRemaining", filter.filterAllRemaining());
     assertTrue("basicFilterNotNull", filter.filterRow());
+    filter.reset();
+    filter.setLatestVersionOnly(false);
+    kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_1);
+    assertTrue("basicFilter5", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE);
+    kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_2);
+    assertTrue("basicFilter5", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE);
+    assertFalse("basicFilterNotNull", filter.filterRow());
   }
 
   private void substrFilterTests(Filter filter) 
@@ -145,7 +152,7 @@
    * @throws Exception
    */
   public void testStop() throws Exception {
-    basicFilterTests(basicFilter);
+    basicFilterTests((SingleColumnValueFilter)basicFilter);
     substrFilterTests(substrFilter);
     regexFilterTests(regexFilter);
   }                               
@@ -156,10 +163,10 @@
    */                       
   public void testSerialization() throws Exception {
     Filter newFilter = serializationTest(basicFilter);
-    basicFilterTests(newFilter);
+    basicFilterTests((SingleColumnValueFilter)newFilter);
     newFilter = serializationTest(substrFilter);
     substrFilterTests(newFilter);
     newFilter = serializationTest(regexFilter);
     regexFilterTests(newFilter);
   }                   
-}
\ No newline at end of file
+}

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java Thu Dec 17 20:26:29 2009
@@ -34,8 +34,8 @@
 import org.apache.hadoop.hbase.util.Bytes;
 
 public class TestStoreScanner extends TestCase {
-
-  final byte [] CF = Bytes.toBytes("cf");
+  private final String CF_STR = "cf";
+  final byte [] CF = Bytes.toBytes(CF_STR);
 
   /**
    * Test utility for building a NavigableSet for scanners.
@@ -50,6 +50,60 @@
     }
     return cols;
   }
+  
+  public void testScanTimeRange() throws IOException {
+    String r1 = "R1";
+    // returns only 1 of these 2 even though same timestamp
+    KeyValue [] kvs = new KeyValue[] {
+        KeyValueTestUtil.create(r1, CF_STR, "a", 1, KeyValue.Type.Put, "dont-care"),
+        KeyValueTestUtil.create(r1, CF_STR, "a", 2, KeyValue.Type.Put, "dont-care"),
+        KeyValueTestUtil.create(r1, CF_STR, "a", 3, KeyValue.Type.Put, "dont-care"),
+        KeyValueTestUtil.create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"),
+        KeyValueTestUtil.create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"),
+    };
+    KeyValueScanner [] scanners = new KeyValueScanner[] {
+      new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
+    };
+    Scan scanSpec = new Scan(Bytes.toBytes(r1));
+    scanSpec.setTimeRange(0, 6);
+    scanSpec.setMaxVersions();
+    StoreScanner scan =
+      new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
+        KeyValue.COMPARATOR, getCols("a"), scanners);
+    List<KeyValue> results = new ArrayList<KeyValue>();
+    assertEquals(true, scan.next(results));
+    assertEquals(5, results.size());
+    assertEquals(kvs[kvs.length - 1], results.get(0));
+    // Scan limited TimeRange
+    scanSpec = new Scan(Bytes.toBytes(r1));
+    scanSpec.setTimeRange(1, 3);
+    scanSpec.setMaxVersions();
+    scan = new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
+      KeyValue.COMPARATOR, getCols("a"), scanners);
+    results = new ArrayList<KeyValue>();
+    assertEquals(true, scan.next(results));
+    assertEquals(2, results.size());
+    // Another range.
+    scanSpec = new Scan(Bytes.toBytes(r1));
+    scanSpec.setTimeRange(5, 10);
+    scanSpec.setMaxVersions();
+    scan = new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
+      KeyValue.COMPARATOR, getCols("a"), scanners);
+    results = new ArrayList<KeyValue>();
+    assertEquals(true, scan.next(results));
+    assertEquals(1, results.size());
+    // See how TimeRange and Versions interact.
+    // Another range.
+    scanSpec = new Scan(Bytes.toBytes(r1));
+    scanSpec.setTimeRange(0, 10);
+    scanSpec.setMaxVersions(3);
+    scan = new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
+      KeyValue.COMPARATOR, getCols("a"), scanners);
+    results = new ArrayList<KeyValue>();
+    assertEquals(true, scan.next(results));
+    assertEquals(3, results.size());
+    
+  }
 
   public void testScanSameTimestamp() throws IOException {
     // returns only 1 of these 2 even though same timestamp
@@ -58,8 +112,7 @@
         KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
     };
     KeyValueScanner [] scanners = new KeyValueScanner[] {
-        new KeyValueScanFixture(KeyValue.COMPARATOR,
-            kvs)
+      new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
     };
 
     Scan scanSpec = new Scan(Bytes.toBytes("R1"));

Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/webapps/master/master.jsp
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/webapps/master/master.jsp?rev=891871&r1=891870&r2=891871&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/webapps/master/master.jsp (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/webapps/master/master.jsp Thu Dec 17 20:26:29 2009
@@ -19,7 +19,7 @@
   Map<byte [], MetaRegion> onlineRegions = master.getOnlineMetaRegions();
   Map<String, HServerInfo> serverToServerInfos =
     master.getServersToServerInfo();
-  int interval = conf.getInt("hbase.regionserver.msginterval", 3000)/1000;
+  int interval = conf.getInt("hbase.regionserver.msginterval", 1000)/1000;
   if (interval == 0) {
       interval = 1;
   }



Mime
View raw message