Return-Path: <commits-return-2078-archive-asf-public=cust-asf.ponee.io@bookkeeper.apache.org>
X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io
Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io
Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183])
	by cust-asf2.ponee.io (Postfix) with ESMTP id 52FC0200D3E
	for <archive-asf-public-internal@cust-asf2.ponee.io>; Thu, 16 Nov 2017 14:06:39 +0100 (CET)
Received: by cust-asf.ponee.io (Postfix)
	id 51783160BE6; Thu, 16 Nov 2017 13:06:39 +0000 (UTC)
Delivered-To: archive-asf-public@cust-asf.ponee.io
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
	by cust-asf.ponee.io (Postfix) with SMTP id 6F6EF160BE5
	for <archive-asf-public@cust-asf.ponee.io>; Thu, 16 Nov 2017 14:06:38 +0100 (CET)
Received: (qmail 6924 invoked by uid 500); 16 Nov 2017 13:06:37 -0000
Mailing-List: contact commits-help@bookkeeper.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:commits-help@bookkeeper.apache.org>
List-Unsubscribe: <mailto:commits-unsubscribe@bookkeeper.apache.org>
List-Post: <mailto:commits@bookkeeper.apache.org>
List-Id: <commits.bookkeeper.apache.org>
Reply-To: bookkeeper-dev@bookkeeper.apache.org
Delivered-To: mailing list commits@bookkeeper.apache.org
Received: (qmail 6915 invoked by uid 99); 16 Nov 2017 13:06:37 -0000
Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70)
    by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 16 Nov 2017 13:06:37 +0000
Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33)
	id 11C1681C2F; Thu, 16 Nov 2017 13:06:37 +0000 (UTC)
Date: Thu, 16 Nov 2017 13:06:37 +0000
To: "commits@bookkeeper.apache.org" <commits@bookkeeper.apache.org>
Subject: [bookkeeper] branch master updated: ISSUE #734:
 bookkeeper-benchmark is broken after protobuf files are moved to
 bookkeeper-proto
MIME-Version: 1.0
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: 8bit
Message-ID: <151083759698.22460.13188956562418627540@gitbox.apache.org>
From: zhaijia@apache.org
X-Git-Host: gitbox.apache.org
X-Git-Repo: bookkeeper
X-Git-Refname: refs/heads/master
X-Git-Reftype: branch
X-Git-Oldrev: 07bb94f25a49c6f28c5b33ce930c0e2a3601db83
X-Git-Newrev: 9b538aff7da5cba15ae59b70e0b0c9bd2464dca3
X-Git-Rev: 9b538aff7da5cba15ae59b70e0b0c9bd2464dca3
X-Git-NotificationType: ref_changed_plus_diff
X-Git-Multimail-Version: 1.5.dev
Auto-Submitted: auto-generated
archived-at: Thu, 16 Nov 2017 13:06:39 -0000

This is an automated email from the ASF dual-hosted git repository.

zhaijia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git


The following commit(s) were added to refs/heads/master by this push:
     new 9b538af  ISSUE #734: bookkeeper-benchmark is broken after protobuf files are moved to bookkeeper-proto
9b538af is described below

commit 9b538aff7da5cba15ae59b70e0b0c9bd2464dca3
Author: Sijie Guo <sijie@apache.org>
AuthorDate: Thu Nov 16 21:06:26 2017 +0800

    ISSUE #734: bookkeeper-benchmark is broken after protobuf files are moved to bookkeeper-proto
    
    Descriptions of the changes in this PR:
    
    Problem:
    
    The issue was introduced after protobuf files are moved to bookkeeper-proto. so the protobuf is not shaded correctly.
    So in bookkeeper-benchmark, it is conflicted with the protobuf version introduced by hadoop.
    
    Solution:
    
    - update the bookkeeper-server pom.xml to shade protobuf correctly
    - remove hadoop dependency from bookkeeper-benchmark.
    
    Author: Sijie Guo <sijie@apache.org>
    
    Reviewers: Ivan Kelly <ivan@ivankelly.net>, Enrico Olivelli <eolivelli@gmail.com>, Jia Zhai <None>
    
    This closes #735 from sijie/protobuf_issue, closes #734
---
 bookkeeper-benchmark/pom.xml                       |  49 --------
 .../apache/bookkeeper/benchmark/MySqlClient.java   | 138 ---------------------
 .../apache/bookkeeper/benchmark/TestClient.java    |  56 ---------
 bookkeeper-server/pom.xml                          |   9 +-
 pom.xml                                            |   7 ++
 5 files changed, 9 insertions(+), 250 deletions(-)

diff --git a/bookkeeper-benchmark/pom.xml b/bookkeeper-benchmark/pom.xml
index 35cf238..b9cb781 100644
--- a/bookkeeper-benchmark/pom.xml
+++ b/bookkeeper-benchmark/pom.xml
@@ -84,24 +84,6 @@
   </build>
   <dependencies>
     <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-log4j12</artifactId>
-      <version>${slf4j.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.zookeeper</groupId>
-      <artifactId>zookeeper</artifactId>
-      <version>${zookeeper.version}</version>
-      <type>jar</type>
-      <scope>compile</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>net.java.dev.javacc</groupId>
-          <artifactId>javacc</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
       <groupId>org.apache.zookeeper</groupId>
       <artifactId>zookeeper</artifactId>
       <version>${zookeeper.version}</version>
@@ -127,17 +109,6 @@
       </exclusions>
     </dependency>
     <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-all</artifactId>
-      <version>${netty.version}</version>
-      <scope>compile</scope>
-    </dependency>
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-      <version>${guava.version}</version>
-    </dependency>
-    <dependency>
       <groupId>org.apache.bookkeeper</groupId>
       <artifactId>bookkeeper-server</artifactId>
       <version>${project.parent.version}</version>
@@ -151,25 +122,5 @@
       <scope>test</scope>
       <type>test-jar</type>
     </dependency>
-    <dependency>
-       <groupId>org.apache.hadoop</groupId>
-       <artifactId>hadoop-common</artifactId>
-       <version>0.23.1</version>
-       <scope>compile</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-      <version>0.23.1</version>
-      <scope>compile</scope>
-      <!-- commons-daemon has a malformed pom, which can cause the build to fail in an
-           environment which uses an artifactory cache. -->
-      <exclusions>
-        <exclusion>
-          <groupId>commons-daemon</groupId>
-          <artifactId>commons-daemon</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
   </dependencies>
 </project>
diff --git a/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/MySqlClient.java b/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/MySqlClient.java
deleted file mode 100644
index f3903f3..0000000
--- a/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/MySqlClient.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.bookkeeper.benchmark;
-
-import static com.google.common.base.Charsets.UTF_8;
-
-import java.io.FileOutputStream;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.HashMap;
-import org.apache.bookkeeper.client.BookKeeper;
-import org.apache.bookkeeper.client.LedgerHandle;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A mysql client performing writes and reads that used for benchmark comparison with BookKeeper.
- */
-public class MySqlClient {
-    static final Logger LOG = LoggerFactory.getLogger(MySqlClient.class);
-
-    BookKeeper x;
-    LedgerHandle lh;
-    Integer entryId;
-    HashMap<Integer, Integer> map;
-
-    FileOutputStream fStream;
-    FileOutputStream fStreamLocal;
-    long start, lastId;
-    Connection con;
-    Statement stmt;
-
-
-    public MySqlClient(String hostport, String user, String pass)
-            throws ClassNotFoundException {
-        entryId = 0;
-        map = new HashMap<Integer, Integer>();
-        Class.forName("com.mysql.jdbc.Driver");
-        // database is named "bookkeeper"
-        String url = "jdbc:mysql://" + hostport + "/bookkeeper";
-        try {
-            con = DriverManager.getConnection(url, user, pass);
-            stmt = con.createStatement();
-            // drop table and recreate it
-            stmt.execute("DROP TABLE IF EXISTS data;");
-            stmt.execute("create table data(transaction_id bigint PRIMARY KEY AUTO_INCREMENT, content TEXT);");
-            LOG.info("Database initialization terminated");
-        } catch (SQLException e) {
-            e.printStackTrace();
-        }
-    }
-
-    public void closeHandle() throws KeeperException, InterruptedException, SQLException {
-        con.close();
-    }
-
-    /**
-     * First parameter is an integer defining the length of the message.
-     * Second parameter is the number of writes.
-     * Third parameter is host:port.
-     * Fourth parameter is username.
-     * Fifth parameter is password.
-     * @param args
-     * @throws ClassNotFoundException
-     * @throws SQLException
-     */
-    public static void main(String[] args) throws ClassNotFoundException, SQLException {
-        int lenght = Integer.parseInt(args[1]);
-        StringBuilder sb = new StringBuilder();
-        while (lenght-- > 0) {
-            sb.append('a');
-        }
-        try {
-            MySqlClient c = new MySqlClient(args[2], args[3], args[4]);
-            c.writeSameEntryBatch(sb.toString().getBytes(UTF_8), Integer.parseInt(args[0]));
-            c.writeSameEntry(sb.toString().getBytes(UTF_8), Integer.parseInt(args[0]));
-            c.closeHandle();
-        } catch (NumberFormatException e) {
-            e.printStackTrace();
-        } catch (InterruptedException e) {
-            e.printStackTrace();
-        } catch (KeeperException e) {
-            e.printStackTrace();
-        }
-
-    }
-
-    /**
-     * Adds data entry to the DB.
-     * @param data the entry to be written, given as a byte array
-     * @param times the number of times the entry should be written on the DB
-     */
-    void writeSameEntryBatch(byte[] data, int times) throws InterruptedException, SQLException {
-        start = System.currentTimeMillis();
-        int count = times;
-        String content = new String(data, UTF_8);
-        System.out.println("Data: " + content + ", " + data.length);
-        while (count-- > 0) {
-            stmt.addBatch("insert into data(content) values(\"" + content + "\");");
-        }
-        LOG.info("Finished writing batch SQL command in ms: " + (System.currentTimeMillis() - start));
-        start = System.currentTimeMillis();
-        stmt.executeBatch();
-        System.out.println("Finished " + times + " writes in ms: " + (System.currentTimeMillis() - start));
-        LOG.info("Ended computation");
-    }
-
-    void writeSameEntry(byte[] data, int times) throws InterruptedException, SQLException {
-        start = System.currentTimeMillis();
-        int count = times;
-        String content = new String(data, UTF_8);
-        System.out.println("Data: " + content + ", " + data.length);
-        while (count-- > 0) {
-            stmt.executeUpdate("insert into data(content) values(\"" + content + "\");");
-        }
-        System.out.println("Finished " + times + " writes in ms: " + (System.currentTimeMillis() - start));
-        LOG.info("Ended computation");
-    }
-
-}
diff --git a/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/TestClient.java b/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/TestClient.java
index 6c1ee05..49f98a7 100644
--- a/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/TestClient.java
+++ b/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/TestClient.java
@@ -48,10 +48,6 @@ import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.cli.PosixParser;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -140,19 +136,6 @@ public class TestClient {
                 for (int i = 0; i < numFiles; i++) {
                     clients.add(new BKClient(handles, data, runfor, cmd.hasOption("sync")));
                 }
-            } else if (target.equals("hdfs")) {
-                FileSystem fs = FileSystem.get(new Configuration());
-                LOG.info("Default replication for HDFS: {}", fs.getDefaultReplication());
-
-                List<FSDataOutputStream> streams = new ArrayList<FSDataOutputStream>();
-                for (int i = 0; i < numFiles; i++) {
-                    String path = cmd.getOptionValue("path", "/foobar");
-                    streams.add(fs.create(new Path(path + runid + "_" + i)));
-                }
-
-                for (int i = 0; i < numThreads; i++) {
-                    clients.add(new HDFSClient(streams, data, runfor));
-                }
             } else if (target.equals("fs")) {
                 List<FileOutputStream> streams = new ArrayList<FileOutputStream>();
                 for (int i = 0; i < numFiles; i++) {
@@ -211,45 +194,6 @@ public class TestClient {
         timeouter.cancel();
     }
 
-    static class HDFSClient implements Callable<Long> {
-        final List<FSDataOutputStream> streams;
-        final byte[] data;
-        final long time;
-        final Random r;
-
-        HDFSClient(List<FSDataOutputStream> streams, byte[] data, long time) {
-            this.streams = streams;
-            this.data = data;
-            this.time = time;
-            this.r = new Random(System.identityHashCode(this));
-        }
-
-        public Long call() {
-            try {
-                long count = 0;
-                long start = System.currentTimeMillis();
-                long stopat = start + time;
-                while (System.currentTimeMillis() < stopat) {
-                    FSDataOutputStream stream = streams.get(r.nextInt(streams.size()));
-                    synchronized (stream) {
-                        stream.write(data);
-                        stream.flush();
-                        stream.hflush();
-                    }
-                    count++;
-                }
-
-                long time = (System.currentTimeMillis() - start);
-                LOG.info("Worker finished processing writes (ms): {} TPT: {} op/s", time,
-                         count / ((double) time / 1000));
-                return count;
-            } catch (IOException ioe) {
-                LOG.error("Exception in worker thread", ioe);
-                return 0L;
-            }
-        }
-    }
-
     static class FileClient implements Callable<Long> {
         final List<FileOutputStream> streams;
         final byte[] data;
diff --git a/bookkeeper-server/pom.xml b/bookkeeper-server/pom.xml
index 2dbef79..606cca0 100644
--- a/bookkeeper-server/pom.xml
+++ b/bookkeeper-server/pom.xml
@@ -179,13 +179,6 @@
     </dependency>
   </dependencies>
   <build>
-    <extensions>
-      <extension>
-        <groupId>kr.motd.maven</groupId>
-        <artifactId>os-maven-plugin</artifactId>
-        <version>1.4.1.Final</version>
-      </extension>
-    </extensions>
     <plugins>
       <plugin>
         <!-- for mini-kdc -->
@@ -214,8 +207,10 @@
               <shadedArtifactAttached>true</shadedArtifactAttached>
               <shadedClassifierName>shaded</shadedClassifierName>
               <minimizeJar>true</minimizeJar>
+              <shadeTestJar>true</shadeTestJar>
               <artifactSet>
                 <includes>
+                  <include>org.apache.bookkeeper:bookkeeper-proto</include>
                   <include>com.google.protobuf:protobuf-java</include>
                   <include>com.google.guava:guava</include>
                 </includes>
diff --git a/pom.xml b/pom.xml
index 13cbabe..fea9dab 100644
--- a/pom.xml
+++ b/pom.xml
@@ -188,6 +188,13 @@
   </dependencies>
 
   <build>
+    <extensions>
+      <extension>
+        <groupId>kr.motd.maven</groupId>
+        <artifactId>os-maven-plugin</artifactId>
+        <version>1.4.1.Final</version>
+      </extension>
+    </extensions>
     <plugins>
       <plugin>
         <groupId>org.codehaus.mojo</groupId>

-- 
To stop receiving notification emails like this one, please contact
['"commits@bookkeeper.apache.org" <commits@bookkeeper.apache.org>'].