Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D999F200C48 for ; Thu, 6 Apr 2017 11:12:37 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id D82DD160B84; Thu, 6 Apr 2017 09:12:37 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 2C478160B83 for ; Thu, 6 Apr 2017 11:12:37 +0200 (CEST) Received: (qmail 59471 invoked by uid 500); 6 Apr 2017 09:12:36 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 59460 invoked by uid 99); 6 Apr 2017 09:12:36 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 06 Apr 2017 09:12:36 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id F19CBDFDAC; Thu, 6 Apr 2017 09:12:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jcamacho@apache.org To: commits@hive.apache.org Message-Id: <8029821de6c745bc85b345f7aa0b6e3d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hive git commit: HIVE-16371: Add bitmap selection strategy for druid storage handler (Slim Bouguerra, reviewed by Jesus Camacho Rodriguez) Date: Thu, 6 Apr 2017 09:12:35 +0000 (UTC) archived-at: Thu, 06 Apr 2017 09:12:38 -0000 Repository: hive Updated Branches: refs/heads/master db24ce722 -> ecabdd3e7 HIVE-16371: Add bitmap selection strategy for druid storage handler (Slim Bouguerra, reviewed by Jesus Camacho Rodriguez) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ecabdd3e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ecabdd3e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ecabdd3e Branch: refs/heads/master Commit: ecabdd3e7eff59fb83b999e4f31533f91fd6391c Parents: db24ce7 Author: Slim Bouguerra Authored: Thu Apr 6 11:01:09 2017 +0200 Committer: Jesus Camacho Rodriguez Committed: Thu Apr 6 11:01:09 2017 +0200 ---------------------------------------------------------------------- .../src/java/org/apache/hadoop/hive/conf/HiveConf.java | 1 + druid-handler/pom.xml | 1 + .../apache/hadoop/hive/druid/io/DruidOutputFormat.java | 11 ++++++++++- .../apache/hadoop/hive/ql/io/TestDruidRecordWriter.java | 8 ++++++-- 4 files changed, 18 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/ecabdd3e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 6de6ed6..7d4a6a0 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2001,6 +2001,7 @@ public class HiveConf extends Configuration { HIVE_DRUID_PASSIVE_WAIT_TIME("hive.druid.passiveWaitTimeMs", 30000, "Wait time in ms default to 30 seconds." ), + HIVE_DRUID_BITMAP_FACTORY_TYPE("hive.druid.bitmap.type", "roaring", new PatternSet("roaring", "concise"), "Coding algorithm use to encode the bitmaps"), // For HBase storage handler HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", true, "Whether writes to HBase should be forced to the write-ahead log. \n" + http://git-wip-us.apache.org/repos/asf/hive/blob/ecabdd3e/druid-handler/pom.xml ---------------------------------------------------------------------- diff --git a/druid-handler/pom.xml b/druid-handler/pom.xml index ffc9d86..6afb5b8 100644 --- a/druid-handler/pom.xml +++ b/druid-handler/pom.xml @@ -329,6 +329,7 @@ org.jdbi:* net.jpountz.lz4:* org.apache.commons:* + org.roaringbitmap:* http://git-wip-us.apache.org/repos/asf/hive/blob/ecabdd3e/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java ---------------------------------------------------------------------- diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java index 40a2022..4385dfe 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java @@ -33,6 +33,9 @@ import io.druid.granularity.QueryGranularity; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.DoubleSumAggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; +import io.druid.segment.IndexSpec; +import io.druid.segment.data.ConciseBitmapSerdeFactory; +import io.druid.segment.data.RoaringBitmapSerdeFactory; import io.druid.segment.indexing.DataSchema; import io.druid.segment.indexing.RealtimeTuningConfig; import io.druid.segment.indexing.granularity.GranularitySpec; @@ -200,6 +203,12 @@ public class DruidOutputFormat implements HiveOutputFormat implements HiveOutputFormat