kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [1/2] kudu git commit: KUDU-1065: [java client] Flexible Partition Pruning
Date Thu, 08 Sep 2016 23:42:39 GMT
Repository: kudu
Updated Branches:
  refs/heads/master e33bac441 -> c7dab48eb


http://git-wip-us.apache.org/repos/asf/kudu/blob/c7dab48e/java/kudu-client/src/test/java/org/apache/kudu/client/TestPartitionPruner.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestPartitionPruner.java
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestPartitionPruner.java
new file mode 100644
index 0000000..b065cdb
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestPartitionPruner.java
@@ -0,0 +1,567 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.kudu.client;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class TestPartitionPruner extends BaseKuduTest {
+
+  /**
+   * Counts the partitions touched by a scan with optional primary key bounds.
+   * The table is assumed to have three INT8 columns as the primary key.
+   *
+   * @param table the table to scan
+   * @param partitions the partitions of the table
+   * @param lowerBoundPrimaryKey the optional lower bound primary key
+   * @param upperBoundPrimaryKey the optional upper bound primary key
+   * @return the number of partitions touched by the scan
+   */
+  private int countPartitionsPrimaryKey(KuduTable table,
+                                        List<Partition> partitions,
+                                        byte[] lowerBoundPrimaryKey,
+                                        byte[] upperBoundPrimaryKey) throws Exception {
+    KuduScanToken.KuduScanTokenBuilder scanBuilder = syncClient.newScanTokenBuilder(table);
+
+    if (lowerBoundPrimaryKey != null) {
+      PartialRow lower = table.getSchema().newPartialRow();
+      for (int i = 0; i < 3; i++) {
+        lower.addByte(i, lowerBoundPrimaryKey[i]);
+      }
+      scanBuilder.lowerBound(lower);
+    }
+
+    if (upperBoundPrimaryKey != null) {
+      PartialRow upper = table.getSchema().newPartialRow();
+      for (int i = 0; i < 3; i++) {
+        upper.addByte(i, upperBoundPrimaryKey[i]);
+      }
+      scanBuilder.exclusiveUpperBound(upper);
+    }
+
+    PartitionPruner pruner = PartitionPruner.create(scanBuilder);
+
+    int scannedPartitions = 0;
+    for (Partition partition : partitions) {
+      if (!pruner.shouldPrune(partition)) scannedPartitions++;
+    }
+
+    // Check that the number of ScanTokens built for the scan matches.
+    assertEquals(scannedPartitions, scanBuilder.build().size());
+    return scannedPartitions;
+  }
+
+  /**
+   * Counts the partitions touched by a scan with predicates.
+   *
+   * @param table the table to scan
+   * @param partitions the partitions of the table
+   * @param predicates the predicates to apply to the scan
+   * @return the number of partitions touched by the scan
+   */
+  private int countPartitions(KuduTable table,
+                              List<Partition> partitions,
+                              KuduPredicate... predicates) throws Exception {
+    KuduScanToken.KuduScanTokenBuilder scanBuilder = syncClient.newScanTokenBuilder(table);
+
+    for (KuduPredicate predicate : predicates) {
+      scanBuilder.addPredicate(predicate);
+    }
+
+    PartitionPruner pruner = PartitionPruner.create(scanBuilder);
+
+    int scannedPartitions = 0;
+    for (Partition partition : partitions) {
+      if (!pruner.shouldPrune(partition)) scannedPartitions++;
+    }
+
+    // Check that the number of ScanTokens built for the scan matches.
+    assertEquals(scannedPartitions, scanBuilder.build().size());
+    return scannedPartitions;
+  }
+
+  /**
+   * Counts the partitions touched by a scan with predicates and optional partition key bounds.
+   *
+   * @param table the table to scan
+   * @param partitions the partitions of the table
+   * @param lowerBoundPartitionKey an optional lower bound partition key
+   * @param upperBoundPartitionKey an optional upper bound partition key
+   * @param predicates the predicates to apply to the scan
+   * @return the number of partitions touched by the scan
+   */
+  private int countPartitions(KuduTable table,
+                              List<Partition> partitions,
+                              byte[] lowerBoundPartitionKey,
+                              byte[] upperBoundPartitionKey,
+                              KuduPredicate... predicates) throws Exception {
+    // Partition key bounds can't be applied to the ScanTokenBuilder.
+    KuduScanner.KuduScannerBuilder scanBuilder = syncClient.newScannerBuilder(table);
+
+    for (KuduPredicate predicate : predicates) {
+      scanBuilder.addPredicate(predicate);
+    }
+
+    if (lowerBoundPartitionKey != null) {
+      scanBuilder.lowerBoundPartitionKeyRaw(lowerBoundPartitionKey);
+    }
+    if (upperBoundPartitionKey != null) {
+      scanBuilder.exclusiveUpperBoundPartitionKeyRaw(upperBoundPartitionKey);
+    }
+
+    PartitionPruner pruner = PartitionPruner.create(scanBuilder);
+
+    int scannedPartitions = 0;
+    for (Partition partition : partitions) {
+      if (!pruner.shouldPrune(partition)) scannedPartitions++;
+    }
+
+    return scannedPartitions;
+  }
+
+  /**
+   * Retrieves the partitions of a table.
+   *
+   * @param table the table
+   * @return the partitions of the table
+   */
+  private List<Partition> getTablePartitions(KuduTable table) {
+    List<Partition> partitions = new ArrayList<>();
+    for (KuduScanToken token : syncClient.newScanTokenBuilder(table).build()) {
+      partitions.add(token.getTablet().getPartition());
+    }
+    return partitions;
+  }
+
+  @Test
+  public void testPrimaryKeyRangePruning() throws Exception {
+    // CREATE TABLE t
+    // (a INT8, b INT8, c INT8)
+    // PRIMARY KEY (a, b, c))
+    // PARTITION BY RANGE (a, b, c)
+    //    (PARTITION                 VALUES < (0, 0, 0),
+    //     PARTITION    (0, 0, 0) <= VALUES < (10, 10, 10)
+    //     PARTITION (10, 10, 10) <= VALUES);
+
+    ArrayList<ColumnSchema> columns = new ArrayList<>(3);
+    columns.add(new ColumnSchema.ColumnSchemaBuilder("a", Type.INT8).key(true).build());
+    columns.add(new ColumnSchema.ColumnSchemaBuilder("b", Type.INT8).key(true).build());
+    columns.add(new ColumnSchema.ColumnSchemaBuilder("c", Type.INT8).key(true).build());
+    Schema schema = new Schema(columns);
+
+    CreateTableOptions tableBuilder = new CreateTableOptions();
+    tableBuilder.setRangePartitionColumns(ImmutableList.of("a", "b", "c"));
+
+    PartialRow split = schema.newPartialRow();
+    split.addByte("a", (byte) 0);
+    split.addByte("b", (byte) 0);
+    split.addByte("c", (byte) 0);
+    tableBuilder.addSplitRow(split);
+    split.addByte("a", (byte) 10);
+    split.addByte("b", (byte) 10);
+    split.addByte("c", (byte) 10);
+    tableBuilder.addSplitRow(split);
+
+    String tableName = "testPrimaryKeyRangePruning-" + System.currentTimeMillis();
+
+    syncClient.createTable(tableName, schema, tableBuilder);
+    KuduTable table = syncClient.openTable(tableName);
+    List<Partition> partitions = getTablePartitions(table);
+
+    byte min = Byte.MIN_VALUE;
+
+    // No bounds
+    assertEquals(3, countPartitionsPrimaryKey(table, partitions, null, null));
+
+    // PK < (-1, min, min)
+    assertEquals(1, countPartitionsPrimaryKey(table, partitions, null,
+                                              new byte[] { -1, min, min }));
+
+    // PK < (10, 10, 10)
+    assertEquals(2, countPartitionsPrimaryKey(table, partitions, null, new byte[] { 10, 10,
10 }));
+
+    // PK < (100, min, min)
+    assertEquals(3, countPartitionsPrimaryKey(table, partitions, null, new byte[] { 100,
min, min }));
+
+    // PK >= (-10, -10, -10)
+    assertEquals(3, countPartitionsPrimaryKey(table, partitions, new byte[] { -10, -10, -10
}, null));
+
+    // PK >= (0, 0, 0)
+    assertEquals(2, countPartitionsPrimaryKey(table, partitions, new byte[] { 0, 0, 0 },
null));
+
+    // PK >= (100, 0, 0)
+    assertEquals(1, countPartitionsPrimaryKey(table, partitions, new byte[] { 100, 0, 0 },
null));
+
+    // PK >= (-10, 0, 0)
+    // PK  < (100, 0, 0)
+    assertEquals(3, countPartitionsPrimaryKey(table, partitions,
+                                              new byte[] { -10, 0, 0 },
+                                              new byte[] { 100, 0, 0 }));
+
+    // PK >= (0, 0, 0)
+    // PK  < (10, 10, 10)
+    assertEquals(1, countPartitionsPrimaryKey(table, partitions,
+                                              new byte[] { 0, 0, 0 },
+                                              new byte[] { 10, 0, 0 }));
+
+    // PK >= (0, 0, 0)
+    // PK  < (10, 10, 11)
+    assertEquals(1, countPartitionsPrimaryKey(table, partitions,
+                                              new byte[] { 0, 0, 0 },
+                                              new byte[] { 10, 0, 0 }));
+
+    // PK < (0, 0, 0)
+    // PK >= (10, 10, 11)
+    assertEquals(0, countPartitionsPrimaryKey(table, partitions,
+                                              new byte[] { 10, 0, 0 },
+                                              new byte[] { 0, 0, 0 }));
+  }
+
+  @Test
+  public void testRangePartitionPruning() throws Exception {
+    // CREATE TABLE t
+    // (a INT8, b STRING, c INT8)
+    // PRIMARY KEY (a, b, c))
+    // DISTRIBUTE BY RANGE(c, b);
+    // PARTITION BY RANGE (a, b, c)
+    //    (PARTITION              VALUES < (0, "m"),
+    //     PARTITION  (0, "m") <= VALUES < (10, "r")
+    //     PARTITION (10, "r") <= VALUES);
+
+    ColumnSchema a = new ColumnSchema.ColumnSchemaBuilder("a", Type.INT8).key(true).build();
+    ColumnSchema b = new ColumnSchema.ColumnSchemaBuilder("b", Type.STRING).key(true).build();
+    ColumnSchema c = new ColumnSchema.ColumnSchemaBuilder("c", Type.INT8).key(true).build();
+    Schema schema = new Schema(ImmutableList.of(a, b, c));
+
+    CreateTableOptions tableBuilder = new CreateTableOptions();
+    tableBuilder.setRangePartitionColumns(ImmutableList.of("c", "b"));
+
+    PartialRow split = schema.newPartialRow();
+    split.addByte("c", (byte) 0);
+    split.addString("b", "m");
+    tableBuilder.addSplitRow(split);
+    split.addByte("c", (byte) 10);
+    split.addString("b", "r");
+    tableBuilder.addSplitRow(split);
+
+    String tableName = "testRangePartitionPruning-" + System.currentTimeMillis();
+    syncClient.createTable(tableName, schema, tableBuilder);
+    KuduTable table = syncClient.openTable(tableName);
+    List<Partition> partitions = getTablePartitions(table);
+
+    // No Predicates
+    assertEquals(3, countPartitions(table, partitions));
+
+    // c < -10
+    assertEquals(1, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.LESS, -10)));
+
+    // c = -10
+    assertEquals(1, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.EQUAL, -10)));
+
+    // c < 10
+    assertEquals(2, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.LESS, 10)));
+
+    // c < 100
+    assertEquals(3, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.LESS, 100)));
+
+
+    // c >= -10
+    assertEquals(3, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.GREATER_EQUAL,
-10)));
+
+    // c >= 0
+    assertEquals(3, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.GREATER_EQUAL,
-10)));
+
+    // c >= 5
+    assertEquals(2, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.GREATER_EQUAL,
5)));
+
+    // c >= 10
+    assertEquals(2, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.GREATER_EQUAL,
10)));
+
+    // c >= 100
+    assertEquals(1, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.GREATER_EQUAL,
100)));
+
+    // c >= -10
+    // c < 0
+    assertEquals(1, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.GREATER_EQUAL,
-10),
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.LESS, 0)));
+
+    // c >= 5
+    // c < 100
+    assertEquals(2, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.GREATER_EQUAL,
5),
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.LESS, 100)));
+
+    // b = ""
+    assertEquals(3, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.EQUAL, "")));
+
+    // b >= "z"
+    assertEquals(3, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.GREATER_EQUAL,
"z")));
+
+    // b < "a"
+    assertEquals(3, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.LESS, "a")));
+
+    // b >= "m"
+    // b < "z"
+    assertEquals(3, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.GREATER_EQUAL,
"m"),
+        KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.LESS, "z")));
+
+    // c >= 10
+    // b >= "r"
+    assertEquals(1, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.GREATER_EQUAL,
10),
+        KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.GREATER_EQUAL,
"r")));
+
+    // c >= 10
+    // b < "r"
+    assertEquals(2, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.GREATER_EQUAL,
10),
+        KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.LESS, "r")));
+
+    // c = 10
+    // b < "r"
+    assertEquals(1, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.EQUAL, 10),
+        KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.LESS, "r")));
+
+    // c < 0
+    // b < "m"
+    assertEquals(1, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.EQUAL, 0),
+        KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.LESS, "m")));
+
+    // c < 0
+    // b < "z"
+    assertEquals(1, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.LESS, 0),
+        KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.LESS, "z")));
+
+    // c = 0
+    // b = "m\0"
+    assertEquals(1, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.EQUAL, 0),
+        KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.EQUAL, "m\0")));
+
+    // c = 0
+    // b < "m"
+    assertEquals(1, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.EQUAL, 0),
+        KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.LESS, "m")));
+
+    // c = 0
+    // b < "m\0"
+    assertEquals(2, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.EQUAL, 0),
+        KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.LESS, "m\0")));
+
+    // c = 0
+    // c = 2
+    assertEquals(0, countPartitions(table, partitions,
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.EQUAL, 0),
+        KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.EQUAL, 2)));
+  }
+
+  @Test
+  public void testHashPartitionPruning() throws Exception {
+    // CREATE TABLE t
+    // (a INT8, b INT8, c INT8)
+    // PRIMARY KEY (a, b, c)
+    // PARTITION BY HASH (a) PARTITIONS 2,
+    //              HASH (b, c) PARTITIONS 2;
+
+    ColumnSchema a = new ColumnSchema.ColumnSchemaBuilder("a", Type.INT8).key(true).build();
+    ColumnSchema b = new ColumnSchema.ColumnSchemaBuilder("b", Type.INT8).key(true).build();
+    ColumnSchema c = new ColumnSchema.ColumnSchemaBuilder("c", Type.INT8).key(true).build();
+    Schema schema = new Schema(ImmutableList.of(a, b, c));
+
+    CreateTableOptions tableBuilder = new CreateTableOptions();
+    tableBuilder.setRangePartitionColumns(new ArrayList<String>());
+    tableBuilder.addHashPartitions(ImmutableList.of("a"), 2);
+    tableBuilder.addHashPartitions(ImmutableList.of("b", "c"), 2);
+
+    String tableName = "testHashPartitionPruning-" + System.currentTimeMillis();
+    syncClient.createTable(tableName, schema, tableBuilder);
+    KuduTable table = syncClient.openTable(tableName);
+    List<Partition> partitions = getTablePartitions(table);
+
+    // No Predicates
+    assertEquals(4, countPartitions(table, partitions));
+
+    // a = 0;
+    assertEquals(2, countPartitions(table, partitions,
+          KuduPredicate.newComparisonPredicate(a, KuduPredicate.ComparisonOp.EQUAL, 0)));
+
+    // a >= 0;
+    assertEquals(4, countPartitions(table, partitions,
+          KuduPredicate.newComparisonPredicate(a, KuduPredicate.ComparisonOp.GREATER_EQUAL,
0)));
+
+    // a >= 0;
+    // a < 1;
+    assertEquals(2, countPartitions(table, partitions,
+          KuduPredicate.newComparisonPredicate(a, KuduPredicate.ComparisonOp.GREATER_EQUAL,
0),
+          KuduPredicate.newComparisonPredicate(a, KuduPredicate.ComparisonOp.LESS, 1)));
+
+    // a >= 0;
+    // a < 2;
+    assertEquals(4, countPartitions(table, partitions,
+          KuduPredicate.newComparisonPredicate(a, KuduPredicate.ComparisonOp.GREATER_EQUAL,
0),
+          KuduPredicate.newComparisonPredicate(a, KuduPredicate.ComparisonOp.LESS, 2)));
+
+    // b = 1;
+    assertEquals(4, countPartitions(table, partitions,
+          KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.EQUAL, 1)));
+
+    // b = 1;
+    // c = 2;
+    assertEquals(2, countPartitions(table, partitions,
+          KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.EQUAL, 1),
+          KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.EQUAL, 2)));
+
+    // a = 0;
+    // b = 1;
+    // c = 2;
+    assertEquals(1, countPartitions(table, partitions,
+          KuduPredicate.newComparisonPredicate(a, KuduPredicate.ComparisonOp.EQUAL, 0),
+          KuduPredicate.newComparisonPredicate(b, KuduPredicate.ComparisonOp.EQUAL, 1),
+          KuduPredicate.newComparisonPredicate(c, KuduPredicate.ComparisonOp.EQUAL, 2)));
+  }
+
+  @Test
+  public void testPruning() throws Exception {
+    // CREATE TABLE timeseries
+    // (host STRING, metric STRING, timestamp TIMESTAMP, value DOUBLE)
+    // PRIMARY KEY (host, metric, time)
+    // DISTRIBUTE BY
+    //    RANGE(time) SPLIT ROWS [(10)],
+    //        (PARTITION       VALUES < 10,
+    //         PARTITION 10 <= VALUES);
+    //    HASH (host, metric) 2 PARTITIONS;
+
+    ColumnSchema host = new ColumnSchema.ColumnSchemaBuilder("host", Type.STRING).key(true).build();
+    ColumnSchema metric = new ColumnSchema.ColumnSchemaBuilder("metric", Type.STRING).key(true).build();
+    ColumnSchema timestamp = new ColumnSchema.ColumnSchemaBuilder("timestamp", Type.TIMESTAMP).key(true).build();
+    ColumnSchema value = new ColumnSchema.ColumnSchemaBuilder("value", Type.DOUBLE).build();
+    Schema schema = new Schema(ImmutableList.of(host, metric, timestamp, value));
+
+    CreateTableOptions tableBuilder = new CreateTableOptions();
+    tableBuilder.setRangePartitionColumns(ImmutableList.of("timestamp"));
+
+    PartialRow split = schema.newPartialRow();
+    split.addLong("timestamp", 10);
+    tableBuilder.addSplitRow(split);
+
+    tableBuilder.addHashPartitions(ImmutableList.of("host", "metric"), 2);
+
+    String tableName = "testPruning-" + System.currentTimeMillis();
+    syncClient.createTable(tableName, schema, tableBuilder);
+    KuduTable table = syncClient.openTable(tableName);
+    List<Partition> partitions = getTablePartitions(table);
+
+    // No Predicates
+    assertEquals(4, countPartitions(table, partitions));
+
+    // host = "a"
+    assertEquals(4, countPartitions(table, partitions,
+          KuduPredicate.newComparisonPredicate(host, KuduPredicate.ComparisonOp.EQUAL, "a")));
+
+    // host = "a"
+    // metric = "a"
+    assertEquals(2, countPartitions(table, partitions,
+          KuduPredicate.newComparisonPredicate(host, KuduPredicate.ComparisonOp.EQUAL, "a"),
+          KuduPredicate.newComparisonPredicate(metric, KuduPredicate.ComparisonOp.EQUAL,
"a")));
+
+    // host = "a"
+    // metric = "a"
+    // timestamp >= 9;
+    assertEquals(2, countPartitions(table, partitions,
+          KuduPredicate.newComparisonPredicate(host, KuduPredicate.ComparisonOp.EQUAL, "a"),
+          KuduPredicate.newComparisonPredicate(metric, KuduPredicate.ComparisonOp.EQUAL,
"a"),
+          KuduPredicate.newComparisonPredicate(timestamp, KuduPredicate.ComparisonOp.GREATER_EQUAL,
9)));
+
+    // host = "a"
+    // metric = "a"
+    // timestamp >= 10;
+    // timestamp < 20;
+    assertEquals(1, countPartitions(table, partitions,
+          KuduPredicate.newComparisonPredicate(host, KuduPredicate.ComparisonOp.EQUAL, "a"),
+          KuduPredicate.newComparisonPredicate(metric, KuduPredicate.ComparisonOp.EQUAL,
"a"),
+          KuduPredicate.newComparisonPredicate(timestamp, KuduPredicate.ComparisonOp.GREATER_EQUAL,
10),
+          KuduPredicate.newComparisonPredicate(timestamp, KuduPredicate.ComparisonOp.LESS,
20)));
+
+    // host = "a"
+    // metric = "a"
+    // timestamp < 10;
+    assertEquals(1, countPartitions(table, partitions,
+          KuduPredicate.newComparisonPredicate(host, KuduPredicate.ComparisonOp.EQUAL, "a"),
+          KuduPredicate.newComparisonPredicate(metric, KuduPredicate.ComparisonOp.EQUAL,
"a"),
+          KuduPredicate.newComparisonPredicate(timestamp, KuduPredicate.ComparisonOp.LESS,
10)));
+
+    // host = "a"
+    // metric = "a"
+    // timestamp >= 10;
+    assertEquals(1, countPartitions(table, partitions,
+          KuduPredicate.newComparisonPredicate(host, KuduPredicate.ComparisonOp.EQUAL, "a"),
+          KuduPredicate.newComparisonPredicate(metric, KuduPredicate.ComparisonOp.EQUAL,
"a"),
+          KuduPredicate.newComparisonPredicate(timestamp, KuduPredicate.ComparisonOp.GREATER_EQUAL,
10)));
+
+    // host = "a"
+    // metric = "a"
+    // timestamp = 10;
+    assertEquals(1, countPartitions(table, partitions,
+          KuduPredicate.newComparisonPredicate(host, KuduPredicate.ComparisonOp.EQUAL, "a"),
+          KuduPredicate.newComparisonPredicate(metric, KuduPredicate.ComparisonOp.EQUAL,
"a"),
+          KuduPredicate.newComparisonPredicate(timestamp, KuduPredicate.ComparisonOp.EQUAL,
10)));
+
+    // partition key < (hash=1)
+    assertEquals(2, countPartitions(table, partitions, new byte[] {}, new byte[] { 0, 0,
0, 1 }));
+
+    // partition key >= (hash=1)
+    assertEquals(2, countPartitions(table, partitions, new byte[] { 0, 0, 0, 1 }, new byte[]
{}));
+
+    // timestamp = 10
+    // partition key < (hash=1)
+    assertEquals(1, countPartitions(table, partitions, new byte[] {}, new byte[] { 0, 0,
0, 1 },
+          KuduPredicate.newComparisonPredicate(timestamp, KuduPredicate.ComparisonOp.EQUAL,
10)));
+
+    // timestamp = 10
+    // partition key >= (hash=1)
+    assertEquals(1, countPartitions(table, partitions, new byte[] { 0, 0, 0, 1 }, new byte[]
{},
+          KuduPredicate.newComparisonPredicate(timestamp, KuduPredicate.ComparisonOp.EQUAL,
10)));
+  }
+}

http://git-wip-us.apache.org/repos/asf/kudu/blob/c7dab48e/java/kudu-client/src/test/java/org/apache/kudu/util/TestByteVec.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/util/TestByteVec.java b/java/kudu-client/src/test/java/org/apache/kudu/util/TestByteVec.java
new file mode 100644
index 0000000..a1720ee
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/util/TestByteVec.java
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.kudu.util;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+
+import static org.junit.Assert.*;
+
+public class TestByteVec {
+  private static final Random RAND = new Random();
+
+  private void assertBytesEqual(byte a, byte b) {
+    if (a != b) throw new AssertionError(String.format("%s != %s", a, b));
+  }
+
+  private List<Byte> random() {
+    return random(RAND.nextInt(1024));
+  }
+
+  private List<Byte> random(int len) {
+    List<Byte> list = new ArrayList<>();
+    for (int i = 0; i < len; i++) {
+      list.add((byte) RAND.nextInt(i + 1));
+    }
+    return Collections.unmodifiableList(list);
+  }
+
+  private void checkByteVec(List<Byte> vals) {
+    ByteVec vec = ByteVec.create();
+    assertEquals(0, vec.len());
+
+    // push
+    for (byte i : vals) {
+      vec.push(i);
+    }
+    assertEquals(vals, vec.asList());
+
+    // withCapacity
+    assertEquals(0, ByteVec.withCapacity(0).capacity());
+    assertEquals(13, ByteVec.withCapacity(13).capacity());
+
+    // wrap
+    assertEquals(vec, ByteVec.wrap(vec.toArray()));
+
+    // clone, equals
+    ByteVec copy = vec.clone();
+    assertEquals(copy, vec);
+
+    // truncate
+    copy.truncate(vec.len() + 1);
+    assertEquals(vals, copy.asList());
+    vec.truncate(copy.len());
+    assertEquals(vals, copy.asList());
+    copy.truncate(vals.size() / 2);
+    assertEquals(vals.subList(0, vals.size() / 2), copy.asList());
+    if (vals.size() > 0) {
+      assertNotEquals(vals, copy.asList());
+    }
+
+    // reserve
+    int unused = copy.capacity() - copy.len();
+
+    copy.reserve(unused);
+    assertEquals(vec.capacity(), copy.capacity());
+
+    copy.reserve(unused + 1);
+    assertTrue(copy.capacity() > vec.capacity());
+
+    // reserveExact
+    unused = copy.capacity() - copy.len();
+    copy.reserveExact(unused + 3);
+    assertEquals(copy.capacity() - copy.len(), unused + 3);
+
+    copy.truncate(0);
+    assertEquals(0, copy.len());
+
+    // shrinkToFit
+    copy.shrinkToFit();
+    assertEquals(0, copy.capacity());
+    vec.shrinkToFit();
+    assertEquals(vec.len(), vec.capacity());
+
+    // get
+    for (int i = 0; i < vals.size(); i++) {
+      assertBytesEqual(vals.get(i), vec.get(i));
+    }
+
+    // set
+    if (vec.len() > 0) {
+      copy = vec.clone();
+      int index = RAND.nextInt(vec.len());
+      copy.set(index, (byte) index);
+      List<Byte> intsCopy = new ArrayList<>(vals);
+      intsCopy.set(index, (byte) index);
+      assertEquals(intsCopy, copy.asList());
+    }
+  }
+
+  @Test
+  public void testByteVec() throws Exception {
+    checkByteVec(random(0));
+    checkByteVec(random(1));
+    checkByteVec(random(2));
+    checkByteVec(random(3));
+    checkByteVec(random(ByteVec.DEFAULT_CAPACITY - 2));
+    checkByteVec(random(ByteVec.DEFAULT_CAPACITY - 1));
+    checkByteVec(random(ByteVec.DEFAULT_CAPACITY));
+    checkByteVec(random(ByteVec.DEFAULT_CAPACITY + 1));
+    checkByteVec(random(ByteVec.DEFAULT_CAPACITY + 2));
+
+    for (int i = 0; i < 100; i++) {
+      checkByteVec(random());
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/kudu/blob/c7dab48e/java/kudu-client/src/test/resources/flags
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/resources/flags b/java/kudu-client/src/test/resources/flags
index 34b73cb..687676f 100644
--- a/java/kudu-client/src/test/resources/flags
+++ b/java/kudu-client/src/test/resources/flags
@@ -1,4 +1,3 @@
---v=1
 --logtostderr
 --never_fsync
 --unlock_experimental_flags

http://git-wip-us.apache.org/repos/asf/kudu/blob/c7dab48e/src/kudu/common/partition_pruner-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/partition_pruner-test.cc b/src/kudu/common/partition_pruner-test.cc
index 72aecf9..9957ee0 100644
--- a/src/kudu/common/partition_pruner-test.cc
+++ b/src/kudu/common/partition_pruner-test.cc
@@ -44,10 +44,10 @@ using std::vector;
 namespace kudu {
 
 void CheckPrunedPartitions(const Schema& schema,
-                            const PartitionSchema& partition_schema,
-                            const vector<Partition> partitions,
-                            const ScanSpec& spec,
-                            size_t remaining_tablets) {
+                           const PartitionSchema& partition_schema,
+                           const vector<Partition> partitions,
+                           const ScanSpec& spec,
+                           size_t remaining_tablets) {
 
   PartitionPruner pruner;
   pruner.Init(schema, partition_schema, spec);
@@ -369,10 +369,6 @@ TEST(TestPartitionPruner, TestRangePruning) {
   // c < 0
   Check({ ColumnPredicate::Range(schema.column(2), &neg_ten, &zero) }, 1);
 
-  // c >= -10
-  // c < 0
-  Check({ ColumnPredicate::Range(schema.column(2), &neg_ten, &zero) }, 1);
-
   // c >= 5
   // c < 100
   Check({ ColumnPredicate::Range(schema.column(2), &five, &hundred) }, 2);
@@ -441,7 +437,7 @@ TEST(TestPartitionPruner, TestRangePruning) {
 
 TEST(TestPartitionPruner, TestHashPruning) {
   // CREATE TABLE t
-  // (a INT8, b STRING, c INT8)
+  // (a INT8, b INT8, c INT8)
   // PRIMARY KEY (a, b, c)
   // DISTRIBUTE BY HASH(a) INTO 2 BUCKETS,
   //               HASH(b, c) INTO 2 BUCKETS;
@@ -523,7 +519,7 @@ TEST(TestPartitionPruner, TestPruning) {
   // CREATE TABLE timeseries
   // (host STRING, metric STRING, time TIMESTAMP, value DOUBLE)
   // PRIMARY KEY (host, metric, time)
-  // DISTRIBUTE BY RANGE(time) SPLIT ROWS [(5), (10)],
+  // DISTRIBUTE BY RANGE(time) SPLIT ROWS [(10)],
   //               HASH(host, metric) INTO 2 BUCKETS;
   Schema schema({ ColumnSchema("host", STRING),
                   ColumnSchema("metric", STRING),
@@ -604,10 +600,10 @@ TEST(TestPartitionPruner, TestPruning) {
 
   // host = "a"
   // metric = "a"
-  // timestamp = 10;
+  // timestamp >= 10;
   Check({ ColumnPredicate::Equality(schema.column(0), &a),
           ColumnPredicate::Equality(schema.column(1), &a),
-          ColumnPredicate::Equality(schema.column(2), &ten) },
+          ColumnPredicate::Range(schema.column(2), &ten, nullptr) },
         "", "",
         1);
 
@@ -626,12 +622,12 @@ TEST(TestPartitionPruner, TestPruning) {
   // partition key >= (hash=1)
   Check({}, string("\0\0\0\1", 4), "", 2);
 
-  // a = 10
+  // timestamp = 10
   // partition key < (hash=1)
   Check({ ColumnPredicate::Equality(schema.column(2), &ten) },
         "", string("\0\0\0\1", 4), 1);
 
-  // a = 10
+  // timestamp = 10
   // partition key >= (hash=1)
   Check({ ColumnPredicate::Equality(schema.column(2), &ten) },
         string("\0\0\0\1", 4), "", 1);


Mime
View raw message