tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jihoon...@apache.org
Subject git commit: TAJO-713: Missing INET4 in UniformRangePartition. (jihoon)
Date Sat, 29 Mar 2014 05:34:37 GMT
Repository: tajo
Updated Branches:
  refs/heads/master ebfcc43b4 -> 85b38348f


TAJO-713: Missing INET4 in UniformRangePartition. (jihoon)


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/85b38348
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/85b38348
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/85b38348

Branch: refs/heads/master
Commit: 85b38348f1da1dec5c187b72ae6ceaf887303b8b
Parents: ebfcc43
Author: Jihoon Son <jihoonson@apache.org>
Authored: Sat Mar 29 14:34:13 2014 +0900
Committer: Jihoon Son <jihoonson@apache.org>
Committed: Sat Mar 29 14:34:13 2014 +0900

----------------------------------------------------------------------
 CHANGES.txt                                     |   2 +
 .../engine/planner/UniformRangePartition.java   |  36 +++-
 .../planner/TestUniformRangePartition.java      | 169 +++++++++++++------
 3 files changed, 152 insertions(+), 55 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/85b38348/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1fdaedb..ac4d325 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -292,6 +292,8 @@ Release 0.8.0 - unreleased
 
   BUG FIXES
 
+    TAJO-713: Missing INET4 in UniformRangePartition. (jihoon)
+
     TAJO-718: A group-by clause with the same columns but aliased causes
     planning error. (hyunsik)
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/85b38348/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java
b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java
index a7df3e8..f6922ed 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java
@@ -29,6 +29,7 @@ import org.apache.tajo.engine.exception.RangeOverflowException;
 import org.apache.tajo.storage.Tuple;
 import org.apache.tajo.storage.TupleRange;
 import org.apache.tajo.storage.VTuple;
+import org.apache.tajo.util.Bytes;
 
 import java.math.BigDecimal;
 import java.math.RoundingMode;
@@ -202,6 +203,25 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
           return candidate.compareTo(new BigDecimal(range.getEnd().get(colId).asChars().charAt(0)))
< 0;
         }
       }
+      case INET4: {
+        int candidateIntVal;
+        byte[] candidateBytesVal = new byte[4];
+        if (sortSpecs[colId].isAscending()) {
+          candidateIntVal = inc.intValue() + last.asInt4();
+          if (candidateIntVal - inc.intValue() != last.asInt4()) {
+            return true;
+          }
+          Bytes.putInt(candidateBytesVal, 0, candidateIntVal);
+          return Bytes.compareTo(range.getEnd().get(colId).asByteArray(), candidateBytesVal)
< 0;
+        } else {
+          candidateIntVal = last.asInt4() - inc.intValue();
+          if (candidateIntVal + inc.intValue() != last.asInt4()) {
+            return true;
+          }
+          Bytes.putInt(candidateBytesVal, 0, candidateIntVal);
+          return Bytes.compareTo(candidateBytesVal, range.getEnd().get(colId).asByteArray())
< 0;
+        }
+      }
     }
     return overflow;
   }
@@ -231,7 +251,8 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
       }
       case TIME:
       case TIMESTAMP:
-      case INT8: {
+      case INT8:
+      case INET4: {
         long candidate = last.asInt8() + inc;
         long end = range.getEnd().get(colId).asInt8();
         reminder = candidate - end;
@@ -407,6 +428,19 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
             end.put(i, DatumFactory.createTimeStampFromMillis(last.get(i).asInt8() + incs[i].longValue()));
           }
           break;
+        case INET4:
+          byte[] ipBytes;
+          if (overflowFlag[i]) {
+            ipBytes = range.getStart().get(i).asByteArray();
+            assert ipBytes.length == 4;
+            end.put(i, DatumFactory.createInet4(ipBytes));
+          } else {
+            int lastVal = last.get(i).asInt4() + incs[i].intValue();
+            ipBytes = new byte[4];
+            Bytes.putInt(ipBytes, 0, lastVal);
+            end.put(i, DatumFactory.createInet4(ipBytes));
+          }
+          break;
         default:
           throw new UnsupportedOperationException(column.getDataType() + " is not supported
yet");
       }

http://git-wip-us.apache.org/repos/asf/tajo/blob/85b38348/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java
b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java
index dc5108a..f4c114f 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java
@@ -18,15 +18,14 @@
 
 package org.apache.tajo.engine.planner;
 
-import org.apache.tajo.catalog.SortSpec;
-import org.junit.Test;
 import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.SortSpec;
 import org.apache.tajo.common.TajoDataTypes.Type;
 import org.apache.tajo.datum.DatumFactory;
-import org.apache.tajo.engine.utils.TupleUtil;
 import org.apache.tajo.storage.Tuple;
 import org.apache.tajo.storage.TupleRange;
 import org.apache.tajo.storage.VTuple;
+import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -38,8 +37,8 @@ public class TestUniformRangePartition {
   @Test
   public void testIncrement1() {
     Schema schema = new Schema()
-    .addColumn("l_returnflag", Type.TEXT)
-    .addColumn("l_linestatus", Type.TEXT);
+        .addColumn("l_returnflag", Type.TEXT)
+        .addColumn("l_linestatus", Type.TEXT);
 
     SortSpec[] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
 
@@ -85,8 +84,8 @@ public class TestUniformRangePartition {
   @Test
   public void testIncrement2() {
     Schema schema = new Schema()
-    .addColumn("l_returnflag", Type.TEXT)
-    .addColumn("l_linestatus", Type.TEXT);
+        .addColumn("l_returnflag", Type.TEXT)
+        .addColumn("l_linestatus", Type.TEXT);
 
     SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
 
@@ -130,9 +129,9 @@ public class TestUniformRangePartition {
   @Test
   public void testIncrement3() {
     Schema schema = new Schema()
-    .addColumn("l_returnflag", Type.TEXT)
-    .addColumn("l_linestatus", Type.TEXT)
-    .addColumn("final", Type.TEXT);
+        .addColumn("l_returnflag", Type.TEXT)
+        .addColumn("l_linestatus", Type.TEXT)
+        .addColumn("final", Type.TEXT);
 
     SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
 
@@ -163,8 +162,8 @@ public class TestUniformRangePartition {
   @Test
   public void testIncrement4() {
     Schema schema = new Schema()
-    .addColumn("l_orderkey", Type.INT8)
-    .addColumn("l_linenumber", Type.INT8);
+        .addColumn("l_orderkey", Type.INT8)
+        .addColumn("l_linenumber", Type.INT8);
 
     SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
 
@@ -190,9 +189,9 @@ public class TestUniformRangePartition {
 
   @Test public void testIncrement5() {
     Schema schema = new Schema()
-    .addColumn("l_orderkey", Type.INT8)
-    .addColumn("l_linenumber", Type.INT8)
-    .addColumn("final", Type.INT8);
+        .addColumn("l_orderkey", Type.INT8)
+        .addColumn("l_linenumber", Type.INT8)
+        .addColumn("final", Type.INT8);
 
     SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
 
@@ -223,9 +222,9 @@ public class TestUniformRangePartition {
   @Test
   public void testIncrement6() {
     Schema schema = new Schema()
-      .addColumn("l_orderkey", Type.FLOAT8)
-      .addColumn("l_linenumber", Type.FLOAT8)
-      .addColumn("final", Type.FLOAT8);
+        .addColumn("l_orderkey", Type.FLOAT8)
+        .addColumn("l_linenumber", Type.FLOAT8)
+        .addColumn("final", Type.FLOAT8);
 
     SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
 
@@ -254,6 +253,39 @@ public class TestUniformRangePartition {
   }
 
   @Test
+  public void testIncrement7() {
+    Schema schema = new Schema()
+        .addColumn("l_orderkey", Type.INET4)
+        .addColumn("l_linenumber", Type.INET4)
+        .addColumn("final", Type.INET4);
+
+    SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
+
+    Tuple s = new VTuple(3);
+    s.put(0, DatumFactory.createInet4("127.0.1.1"));
+    s.put(1, DatumFactory.createInet4("127.0.0.1"));
+    s.put(2, DatumFactory.createInet4("128.0.0.253"));
+    Tuple e = new VTuple(3);
+    e.put(0, DatumFactory.createInet4("127.0.1.4")); // 4
+    e.put(1, DatumFactory.createInet4("127.0.0.2")); // 2
+    e.put(2, DatumFactory.createInet4("128.0.0.255")); //x3 = 24
+
+    TupleRange expected = new TupleRange(sortSpecs, s, e);
+
+    UniformRangePartition partitioner = new UniformRangePartition(expected, sortSpecs);
+    assertEquals(24, partitioner.getTotalCardinality().longValue());
+
+    Tuple beforeOverflow = partitioner.increment(s, 5, 2);
+    assertTrue("127.0.1.1".equals(beforeOverflow.get(0).asChars()));
+    assertTrue("127.0.0.2".equals(beforeOverflow.get(1).asChars()));
+    assertTrue("128.0.0.255".equals(beforeOverflow.get(2).asChars()));
+    Tuple overflow = partitioner.increment(beforeOverflow, 1, 2);
+    assertTrue("127.0.1.2".equals(overflow.get(0).asChars()));
+    assertTrue("127.0.0.1".equals(overflow.get(1).asChars()));
+    assertTrue("128.0.0.253".equals(overflow.get(2).asChars()));
+  }
+
+  @Test
   public void testPartition() {
     Schema schema = new Schema();
     schema.addColumn("l_returnflag", Type.TEXT);
@@ -286,8 +318,8 @@ public class TestUniformRangePartition {
   @Test
   public void testPartitionForOnePartNum() {
     Schema schema = new Schema()
-      .addColumn("l_returnflag", Type.TEXT)
-      .addColumn("l_linestatus", Type.TEXT);
+        .addColumn("l_returnflag", Type.TEXT)
+        .addColumn("l_linestatus", Type.TEXT);
 
     SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
 
@@ -308,8 +340,8 @@ public class TestUniformRangePartition {
   @Test
   public void testPartitionForOnePartNumWithOneOfTheValueNull() {
     Schema schema = new Schema()
-            .addColumn("l_returnflag", Type.TEXT)
-            .addColumn("l_linestatus", Type.TEXT);
+        .addColumn("l_returnflag", Type.TEXT)
+        .addColumn("l_linestatus", Type.TEXT);
 
     SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
 
@@ -321,7 +353,7 @@ public class TestUniformRangePartition {
     e.put(1, DatumFactory.createNullDatum());
     TupleRange expected = new TupleRange(sortSpecs, s, e);
     RangePartitionAlgorithm partitioner =
-            new UniformRangePartition(expected, sortSpecs, true);
+        new UniformRangePartition(expected, sortSpecs, true);
     TupleRange [] ranges = partitioner.partition(1);
 
     assertEquals(expected, ranges[0]);
@@ -330,8 +362,8 @@ public class TestUniformRangePartition {
   @Test
   public void testPartitionForOnePartNumWithBothValueNull() {
     Schema schema = new Schema()
-            .addColumn("l_returnflag", Type.TEXT)
-            .addColumn("l_linestatus", Type.TEXT);
+        .addColumn("l_returnflag", Type.TEXT)
+        .addColumn("l_linestatus", Type.TEXT);
 
     SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
 
@@ -343,39 +375,68 @@ public class TestUniformRangePartition {
     e.put(1, DatumFactory.createNullDatum());
     TupleRange expected = new TupleRange(sortSpecs, s, e);
     RangePartitionAlgorithm partitioner =
-            new UniformRangePartition(expected, sortSpecs, true);
+        new UniformRangePartition(expected, sortSpecs, true);
     TupleRange [] ranges = partitioner.partition(1);
 
     assertEquals(expected, ranges[0]);
   }
 
-    @Test
-    public void testPartitionWithNull() {
-        Schema schema = new Schema();
-        schema.addColumn("l_returnflag", Type.TEXT);
-        schema.addColumn("l_linestatus", Type.TEXT);
-
-        SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
-
-        Tuple s = new VTuple(2);
-        s.put(0, DatumFactory.createNullDatum());
-        s.put(1, DatumFactory.createText("F"));
-        Tuple e = new VTuple(2);
-        e.put(0, DatumFactory.createNullDatum());
-        e.put(1, DatumFactory.createText("O"));
-        TupleRange expected = new TupleRange(sortSpecs, s, e);
-        RangePartitionAlgorithm partitioner
-                = new UniformRangePartition(expected, sortSpecs, true);
-        TupleRange [] ranges = partitioner.partition(10);
-
-
-        TupleRange prev = null;
-        for (TupleRange r : ranges) {
-            if (prev == null) {
-                prev = r;
-            } else {
-                assertTrue(prev.compareTo(r) > 0);
-            }
-        }
+  @Test
+  public void testPartitionWithNull() {
+    Schema schema = new Schema();
+    schema.addColumn("l_returnflag", Type.TEXT);
+    schema.addColumn("l_linestatus", Type.TEXT);
+
+    SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
+
+    Tuple s = new VTuple(2);
+    s.put(0, DatumFactory.createNullDatum());
+    s.put(1, DatumFactory.createText("F"));
+    Tuple e = new VTuple(2);
+    e.put(0, DatumFactory.createNullDatum());
+    e.put(1, DatumFactory.createText("O"));
+    TupleRange expected = new TupleRange(sortSpecs, s, e);
+    RangePartitionAlgorithm partitioner
+        = new UniformRangePartition(expected, sortSpecs, true);
+    TupleRange [] ranges = partitioner.partition(10);
+
+
+    TupleRange prev = null;
+    for (TupleRange r : ranges) {
+      if (prev == null) {
+        prev = r;
+      } else {
+        assertTrue(prev.compareTo(r) > 0);
+      }
+    }
+  }
+
+  @Test
+  public void testPartitionWithINET4() {
+    Schema schema = new Schema();
+    schema.addColumn("l_returnflag", Type.INET4);
+    schema.addColumn("l_linestatus", Type.INET4);
+
+    SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
+
+    Tuple s = new VTuple(2);
+    s.put(0, DatumFactory.createInet4("127.0.1.10"));
+    s.put(1, DatumFactory.createInet4("127.0.2.10"));
+    Tuple e = new VTuple(2);
+    e.put(0, DatumFactory.createInet4("127.0.1.20"));
+    e.put(1, DatumFactory.createInet4("127.0.2.20"));
+    TupleRange expected = new TupleRange(sortSpecs, s, e);
+    RangePartitionAlgorithm partitioner
+        = new UniformRangePartition(expected, sortSpecs, true);
+    TupleRange [] ranges = partitioner.partition(10);
+
+    TupleRange prev = null;
+    for (TupleRange r : ranges) {
+      if (prev == null) {
+        prev = r;
+      } else {
+        assertTrue(prev.compareTo(r) < 0);
+      }
     }
+  }
 }


Mime
View raw message