Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id F1132200BA7 for ; Fri, 21 Oct 2016 16:29:12 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id EFE57160AE8; Fri, 21 Oct 2016 14:29:12 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 24DB1160AE0 for ; Fri, 21 Oct 2016 16:29:11 +0200 (CEST) Received: (qmail 91936 invoked by uid 500); 21 Oct 2016 14:29:11 -0000 Mailing-List: contact commits-help@eagle.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@eagle.incubator.apache.org Delivered-To: mailing list commits@eagle.incubator.apache.org Received: (qmail 91927 invoked by uid 99); 21 Oct 2016 14:29:11 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 21 Oct 2016 14:29:11 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id D2D04180993 for ; Fri, 21 Oct 2016 14:29:10 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -6.219 X-Spam-Level: X-Spam-Status: No, score=-6.219 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-2.999] autolearn=disabled Received: from mx1-lw-us.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id UaZlGSmYtW4J for ; Fri, 21 Oct 2016 14:29:07 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-us.apache.org (ASF Mail Server at mx1-lw-us.apache.org) with SMTP id 245C85FBF0 for ; Fri, 21 Oct 2016 14:29:07 +0000 (UTC) Received: (qmail 89966 invoked by uid 99); 21 Oct 2016 14:28:09 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 21 Oct 2016 14:28:09 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 2EAD5DFE80; Fri, 21 Oct 2016 14:28:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hao@apache.org To: commits@eagle.incubator.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: incubator-eagle git commit: Minor: Expose several async producer config Date: Fri, 21 Oct 2016 14:28:09 +0000 (UTC) archived-at: Fri, 21 Oct 2016 14:29:13 -0000 Repository: incubator-eagle Updated Branches: refs/heads/master 281aa52b0 -> 3980dcb4c Minor: Expose several async producer config Author: Zhao, Qingwen Closes #549 from qingwen220/kafkaProducer. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/3980dcb4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/3980dcb4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/3980dcb4 Branch: refs/heads/master Commit: 3980dcb4c34f024e8247747a48f5966438415c82 Parents: 281aa52 Author: Zhao, Qingwen Authored: Fri Oct 21 22:27:55 2016 +0800 Committer: Hao Chen Committed: Fri Oct 21 22:27:55 2016 +0800 ---------------------------------------------------------------------- .../apache/eagle/app/sink/KafkaStreamSink.java | 10 ++++++ .../eagle/app/sink/KafkaStreamSinkConfig.java | 36 ++++++++++++++++++++ ...ecurity.auditlog.HdfsAuditLogAppProvider.xml | 25 ++++++++++++++ .../src/main/resources/application.conf | 4 +++ 4 files changed, 75 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3980dcb4/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java index 2ac4779..e2a4b70 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java @@ -49,6 +49,11 @@ public class KafkaStreamSink extends StormStreamSink { properties.put("metadata.broker.list", config.getBrokerList()); properties.put("serializer.class", config.getSerializerClass()); properties.put("key.serializer.class", config.getKeySerializerClass()); + // new added properties for async producer + properties.put("producer.type", config.getProducerType()); + properties.put("batch.num.messages", config.getNumBatchMessages()); + properties.put("request.required.acks", config.getRequestRequiredAcks()); + properties.put("queue.buffering.max.ms", config.getMaxQueueBufferMs()); ProducerConfig producerConfig = new ProducerConfig(properties); producer = new Producer(producerConfig); } @@ -97,6 +102,11 @@ public class KafkaStreamSink extends StormStreamSink { desc.setBrokerList(config.getString("dataSinkConfig.brokerList")); desc.setSerializerClass(config.getString("dataSinkConfig.serializerClass")); desc.setKeySerializerClass(config.getString("dataSinkConfig.keySerializerClass")); + // new added properties for async producer + desc.setNumBatchMessages(config.getString("dataSinkConfig.numBatchMessages")); + desc.setProducerType(config.getString("dataSinkConfig.producerType")); + desc.setMaxQueueBufferMs(config.getString("dataSinkConfig.maxQueueBufferMs")); + desc.setRequestRequiredAcks(config.getString("dataSinkConfig.requestRequiredAcks")); return desc; } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3980dcb4/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSinkConfig.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSinkConfig.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSinkConfig.java index 9d6a0ab..d5479df 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSinkConfig.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSinkConfig.java @@ -23,6 +23,10 @@ public class KafkaStreamSinkConfig implements StreamSinkConfig { private String brokerList; private String serializerClass; private String keySerializerClass; + private String numBatchMessages; + private String maxQueueBufferMs; + private String producerType; + private String requestRequiredAcks; public String getTopicId() { return topicId; @@ -56,6 +60,38 @@ public class KafkaStreamSinkConfig implements StreamSinkConfig { this.keySerializerClass = keySerializerClass; } + public String getNumBatchMessages() { + return numBatchMessages; + } + + public void setNumBatchMessages(String numBatchMessages) { + this.numBatchMessages = numBatchMessages; + } + + public String getMaxQueueBufferMs() { + return maxQueueBufferMs; + } + + public void setMaxQueueBufferMs(String maxQueueBufferMs) { + this.maxQueueBufferMs = maxQueueBufferMs; + } + + public String getProducerType() { + return producerType; + } + + public void setProducerType(String producerType) { + this.producerType = producerType; + } + + public String getRequestRequiredAcks() { + return requestRequiredAcks; + } + + public void setRequestRequiredAcks(String requestRequiredAcks) { + this.requestRequiredAcks = requestRequiredAcks; + } + @Override public String getType() { return "KAFKA"; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3980dcb4/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml index 9f10fdc..2c02a8f 100644 --- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml +++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml @@ -149,6 +149,31 @@ serializer class Kafka message key + + dataSinkConfig.producerType + dataSinkConfig.keySerializerClass + async + whether the messages are sent asynchronously in a background thread + + + dataSinkConfig.numBatchMessages + dataSinkConfig.numBatchMessages + 4096 + number of messages to send in one batch when using async mode + + + dataSinkConfig.maxQueueBufferMs + dataSinkConfig.maxQueueBufferMs + 5000 + maximum time to buffer data when using async mode + + + dataSinkConfig.requestRequiredAcks + dataSinkConfig.requestRequiredAcks + 0 + value controls when a produce request is considered completed + + fs.defaultFS http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/3980dcb4/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf index c8bbcb1..be357ad 100644 --- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf +++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf @@ -41,5 +41,9 @@ "brokerList" : "server.eagle.apache.org:6667", "serializerClass" : "kafka.serializer.StringEncoder", "keySerializerClass" : "kafka.serializer.StringEncoder" + "producerType" : "async", + "numBatchMessages" : "4096", + "maxQueueBufferMs" : "5000", + "requestRequiredAcks" : "0" } }