crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject git commit: CRUNCH-386: HBase 0.98.1 upgrade/fixes
Date Thu, 01 May 2014 17:15:49 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 2c9e8306c -> c88ce4718


CRUNCH-386: HBase 0.98.1 upgrade/fixes


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/c88ce471
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/c88ce471
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/c88ce471

Branch: refs/heads/master
Commit: c88ce47187b0a06ceba1e12ddee3077bf3f16b36
Parents: 2c9e830
Author: Josh Wills <jwills@apache.org>
Authored: Wed Apr 30 09:50:57 2014 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Thu May 1 09:56:42 2014 -0700

----------------------------------------------------------------------
 .../org/apache/crunch/io/hbase/HFileSourceIT.java     |  2 ++
 .../org/apache/crunch/io/hbase/HFileTargetIT.java     |  6 +++---
 .../org/apache/crunch/io/hbase/HFileInputFormat.java  |  2 +-
 .../crunch/io/hbase/HFileOutputFormatForCrunch.java   | 14 +++++++++-----
 .../apache/crunch/io/hbase/HFileReaderFactory.java    |  2 +-
 .../java/org/apache/crunch/io/hbase/HFileUtils.java   |  5 +----
 pom.xml                                               |  4 ++--
 7 files changed, 19 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java
index 05c6a42..e82102b 100644
--- a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java
+++ b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileContext;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Before;
 import org.junit.Rule;
@@ -304,6 +305,7 @@ public class HFileSourceIT implements Serializable {
       w = HFile.getWriterFactory(conf, new CacheConfig(conf))
           .withPath(fs, inputPath)
           .withComparator(KeyValue.COMPARATOR)
+          .withFileContext(new HFileContext())
           .create();
       for (KeyValue kv : sortedKVs) {
         w.append(kv);

http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
index 7dd035e..25cec98 100644
--- a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
+++ b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
@@ -259,8 +259,8 @@ public class HFileTargetIT implements Serializable {
       }
       HFile.Reader reader = null;
       try {
-        reader = HFile.createReader(fs, f, new CacheConfig(conf));
-        assertEquals(DataBlockEncoding.PREFIX, reader.getEncodingOnDisk());
+        reader = HFile.createReader(fs, f, new CacheConfig(conf), conf);
+        assertEquals(DataBlockEncoding.PREFIX, reader.getDataBlockEncoding());
       } finally {
         reader.close();
       }
@@ -328,7 +328,7 @@ public class HFileTargetIT implements Serializable {
           fs,
           f,
           new CacheConfig(fs.getConf()),
-          DataBlockEncoding.NONE);
+          fs.getConf());
       StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
       scanner.seek(fakeKV); // have to call seek of each underlying scanner, otherwise KeyValueHeap
won't work
       scanners.add(scanner);

http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java
index 9ced0ac..43005ab 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java
@@ -96,7 +96,7 @@ public class HFileInputFormat extends FileInputFormat<NullWritable, KeyValue>
{
       Path path = fileSplit.getPath();
       FileSystem fs = path.getFileSystem(conf);
       LOG.info("Initialize HFileRecordReader for " + path);
-      this.in = HFile.createReader(fs, path, new CacheConfig(conf));
+      this.in = HFile.createReader(fs, path, new CacheConfig(conf), conf);
 
       // The file info must be loaded before the scanner can be used.
       // This seems like a bug in HBase, but it's easily worked around.

http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java
b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java
index 9d5f6ed..ad68da4 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileContext;
 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
 import org.apache.hadoop.hbase.regionserver.HStore;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -89,12 +90,8 @@ public class HFileOutputFormatForCrunch extends FileOutputFormat<Object,
KeyValu
     LOG.info("HColumnDescriptor: " + hcol.toString());
     final HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
         .withPath(fs, outputPath)
-        .withBlockSize(hcol.getBlocksize())
-        .withCompression(hcol.getCompression())
         .withComparator(KeyValue.COMPARATOR)
-        .withDataBlockEncoder(new HFileDataBlockEncoderImpl(hcol.getDataBlockEncoding()))
-        .withChecksumType(HStore.getChecksumType(conf))
-        .withBytesPerChecksum(HStore.getBytesPerChecksum(conf))
+        .withFileContext(getContext(hcol))
         .create();
 
     return new RecordWriter<Object, KeyValue>() {
@@ -124,4 +121,11 @@ public class HFileOutputFormatForCrunch extends FileOutputFormat<Object,
KeyValu
       }
     };
   }
+
+  private HFileContext getContext(HColumnDescriptor desc) {
+    HFileContext ctxt = new HFileContext();
+    ctxt.setDataBlockEncoding(desc.getDataBlockEncoding());
+    ctxt.setCompression(desc.getCompression());
+    return ctxt;
+  }
 }

http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java
b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java
index f5db516..6189775 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java
@@ -39,7 +39,7 @@ public class HFileReaderFactory implements FileReaderFactory<KeyValue>
{
     Configuration conf = fs.getConf();
     CacheConfig cacheConfig = new CacheConfig(conf);
     try {
-      HFile.Reader hfr = HFile.createReader(fs, path, cacheConfig);
+      HFile.Reader hfr = HFile.createReader(fs, path, cacheConfig, conf);
       HFileScanner scanner = hfr.getScanner(
           conf.getBoolean(HFILE_SCANNER_CACHE_BLOCKS, false),
           conf.getBoolean(HFILE_SCANNER_PREAD, false));

http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java
index cb82dd8..63d2286 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java
@@ -86,7 +86,7 @@ public final class HFileUtils {
       if ((cmp = compareType(l, r)) != 0) {
         return cmp;
       }
-      return compareMemstoreTS(l, r);
+      return 0;
     }
 
     private int compareFamily(KeyValue l, KeyValue r) {
@@ -111,9 +111,6 @@ public final class HFileUtils {
       return (int) r.getType() - (int) l.getType();
     }
 
-    private int compareMemstoreTS(KeyValue l, KeyValue r) {
-      return Longs.compare(l.getMemstoreTS(), r.getMemstoreTS());
-    }
   };
 
   private static class FilterByFamilyFn extends FilterFn<KeyValue> {

http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index aafe113..1787830 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,7 +88,7 @@ under the License.
 
     <!-- Can be overridden by hadoop-2 profile, but these are the default values -->
     <hadoop.version>1.1.2</hadoop.version>
-    <hbase.version>0.96.0-hadoop1</hbase.version>
+    <hbase.version>0.98.1-hadoop1</hbase.version>
     <hbase.midfix>hadoop1</hbase.midfix>
 
     <scala.base.version>2.10</scala.base.version>
@@ -457,7 +457,7 @@ under the License.
       </activation>
       <properties>
         <hadoop.version>2.2.0</hadoop.version>
-        <hbase.version>0.96.0-hadoop2</hbase.version>
+        <hbase.version>0.98.1-hadoop2</hbase.version>
         <commons-lang.version>2.5</commons-lang.version>
         <hbase.midfix>hadoop2</hbase.midfix>
       </properties>


Mime
View raw message