phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maryann...@apache.org
Subject [01/50] [abbrv] phoenix git commit: PHOENIX-2649 - GC/OOM during BulkLoad
Date Wed, 24 Feb 2016 21:13:19 GMT
Repository: phoenix
Updated Branches:
  refs/heads/calcite 69a2f2437 -> 9ac854ae5


PHOENIX-2649 - GC/OOM during BulkLoad


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/12f6a6f4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/12f6a6f4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/12f6a6f4

Branch: refs/heads/calcite
Commit: 12f6a6f48c4a4739e09f1842d885d53e2e5550e2
Parents: 5c25a72
Author: ravimagham <ravi.magham@bazaarvoice.com>
Authored: Wed Feb 3 14:15:16 2016 -0800
Committer: ravimagham <ravi.magham@bazaarvoice.com>
Committed: Wed Feb 3 14:15:16 2016 -0800

----------------------------------------------------------------------
 .../mapreduce/bulkload/TableRowkeyPair.java     | 41 +++---------
 .../mapreduce/bulkload/TestTableRowkeyPair.java | 67 ++++++++++++++++++++
 2 files changed, 75 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/12f6a6f4/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
index 412226f..e3032be 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
@@ -84,6 +84,13 @@ public class TableRowkeyPair implements WritableComparable<TableRowkeyPair>
{
         WritableUtils.writeString(output,tableName);
         rowkey.write(output);
     }
+    
+    @Override
+    public int hashCode() {
+        int result = this.tableName.hashCode();
+        result = 31 * result + this.rowkey.hashCode();
+        return result;
+    }
 
     @Override
     public int compareTo(TableRowkeyPair other) {
@@ -95,40 +102,8 @@ public class TableRowkeyPair implements WritableComparable<TableRowkeyPair>
{
         }
     }
     
-    /** Comparator optimized for <code>TableRowkeyPair</code>. */
-    public static class Comparator extends WritableComparator {
-        private BytesWritable.Comparator comparator = new BytesWritable.Comparator();
-        
-        public Comparator() {
-            super(TableRowkeyPair.class);
-        }
-
-        @Override
-        public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
-            try {
-                int vintL1 = WritableUtils.decodeVIntSize(b1[s1]);
-                int vintL2 = WritableUtils.decodeVIntSize(b2[s2]);
-                int strL1 = readVInt(b1, s1);
-                int strL2 = readVInt(b2, s2);
-                int cmp = compareBytes(b1, s1 + vintL1, strL1, b2, s2 + vintL2, strL2);
-                if (cmp != 0) {
-                  return cmp;
-                }
-                int vintL3 = WritableUtils.decodeVIntSize(b1[s1 + vintL1 + strL1]);
-                int vintL4 = WritableUtils.decodeVIntSize(b2[s2 + vintL2 + strL2]);
-                int strL3 = readVInt(b1, s1 + vintL1 + strL1);
-                int strL4 = readVInt(b2, s2 + vintL2 + strL2);
-                return comparator.compare(b1, s1 + vintL1 + strL1 + vintL3, strL3, b2, s2
-                    + vintL2 + strL2 + vintL4, strL4);
-                
-            } catch(Exception ex) {
-                throw new IllegalArgumentException(ex);
-            }
-        }
-    }
- 
     static { 
-        WritableComparator.define(TableRowkeyPair.class, new Comparator());
+        WritableComparator.define(TableRowkeyPair.class, new BytesWritable.Comparator());
     }
 
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/12f6a6f4/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java
b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java
new file mode 100644
index 0000000..1fee4bb
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce.bulkload;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.BytesWritable;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests for {@linkplain TableRowkeyPair} 
+ */
+public class TestTableRowkeyPair {
+
+    @Test
+    public void testRowkeyPair() throws IOException {
+        testsRowsKeys("first", "aa", "first", "aa", 0);
+        testsRowsKeys("first", "aa", "first", "ab", -1);
+        testsRowsKeys("second", "aa", "first", "aa", 1);
+        testsRowsKeys("first", "aa", "first", "aaa", -1);
+    }
+    
+    private void testsRowsKeys(String aTable, String akey, String bTable, String bkey, int
expectedSignum) throws IOException {
+        
+        final ImmutableBytesWritable arowkey = new ImmutableBytesWritable(Bytes.toBytes(akey));
+        TableRowkeyPair pair1 = new TableRowkeyPair(aTable, arowkey);
+        
+        ImmutableBytesWritable browkey = new ImmutableBytesWritable(Bytes.toBytes(bkey));
+        TableRowkeyPair pair2 = new TableRowkeyPair(bTable, browkey);
+        
+        BytesWritable.Comparator comparator = new BytesWritable.Comparator();
+        try( ByteArrayOutputStream baosA = new ByteArrayOutputStream();
+             ByteArrayOutputStream baosB = new ByteArrayOutputStream()) {
+            
+            pair1.write(new DataOutputStream(baosA));
+            pair2.write(new DataOutputStream(baosB));
+            Assert.assertEquals(expectedSignum , signum(pair1.compareTo(pair2)));
+            Assert.assertEquals(expectedSignum , signum(comparator.compare(baosA.toByteArray(),
0, baosA.size(), baosB.toByteArray(), 0, baosB.size())));
+            Assert.assertEquals(expectedSignum, -signum(comparator.compare(baosB.toByteArray(),
0, baosB.size(), baosA.toByteArray(), 0, baosA.size())));
+        }
+
+    }
+    
+    private int signum(int i) {
+        return i > 0 ? 1: (i == 0 ? 0: -1);
+    }
+}


Mime
View raw message