kafka-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ewe...@apache.org
Subject [kafka] branch trunk updated: MINOR: Support choosing different JVMs when running integration tests
Date Fri, 11 Jan 2019 23:12:20 GMT
This is an automated email from the ASF dual-hosted git repository.

ewencp pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git


The following commit(s) were added to refs/heads/trunk by this push:
     new cc33511  MINOR: Support choosing different JVMs when running integration tests
cc33511 is described below

commit cc33511e9a0a1493ef89afefb7df089ca546687e
Author: Xi Yang <xi@confluent.io>
AuthorDate: Fri Jan 11 15:11:55 2019 -0800

    MINOR: Support choosing different JVMs when running integration tests
    
    + Add a parameter to the ducktap-ak to control the OpenJDK base image.
    + Fix a few issues of using OpenJDK:11 as the base image.
    
    *More detailed description of your change,
    if necessary. The PR title and PR message become
    the squashed commit message, so use a separate
    comment to ping reviewers.*
    
    *Summary of testing strategy (including rationale)
    for the feature or bug fix. Unit and/or integration
    tests are expected for any behaviour change and
    system tests should be considered for larger changes.*
    
    Author: Xi Yang <xi@confluent.io>
    
    Reviewers: Ewen Cheslack-Postava <ewen@confluent.io>
    
    Closes #6071 from yangxi/ducktape-jdk
---
 tests/README.md         |  4 ++++
 tests/docker/Dockerfile | 13 ++++++++-----
 tests/docker/ducker-ak  |  9 +++++++--
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index f42b28a..6c20553 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -36,6 +36,10 @@ TC_PATHS="tests/kafkatest/tests/client/pluggable_test.py::PluggableConsumerTest"
 ```
 TC_PATHS="tests/kafkatest/tests/client/pluggable_test.py::PluggableConsumerTest.test_start_stop"
bash tests/docker/run_tests.sh
 ```
+* Run tests with a different JVM
+```
+bash tests/docker/ducker-ak up -j 'openjdk:11'; tests/docker/run_tests.sh
+```
 
 * Notes
   - The scripts to run tests creates and destroys docker network named *knw*.
diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile
index e5cf439..68efaee 100644
--- a/tests/docker/Dockerfile
+++ b/tests/docker/Dockerfile
@@ -13,7 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM openjdk:8
+ARG jdk_version=openjdk:8
+FROM $jdk_version
 
 MAINTAINER Apache Kafka dev@kafka.apache.org
 VOLUME ["/opt/kafka-dev"]
@@ -31,12 +32,14 @@ ARG ducker_creator=default
 LABEL ducker.creator=$ducker_creator
 
 # Update Linux and install necessary utilities.
-RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils
openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config
libfuse-dev && apt-get -y clean
-RUN pip install -U pip==9.0.3 setuptools && pip install --upgrade cffi virtualenv
pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip install --upgrade ducktape==0.7.1
+RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils
openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config
libfuse-dev  && apt-get -y clean
+RUN python -m pip install -U pip==9.0.3;
+RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34
&& pip install --upgrade ducktape==0.7.1
 
 # Set up ssh
 COPY ./ssh-config /root/.ssh/config
-RUN ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa && cp -f /root/.ssh/id_rsa.pub
/root/.ssh/authorized_keys
+# NOTE: The paramiko library supports the PEM-format private key, but does not support the
RFC4716 format.
+RUN ssh-keygen -m PEM -q -t rsa -N '' -f /root/.ssh/id_rsa && cp -f /root/.ssh/id_rsa.pub
/root/.ssh/authorized_keys
 
 # Install binary test dependencies.
 # we use the same versions as in vagrant/base.sh
@@ -71,7 +74,7 @@ RUN apt-get install fuse
 RUN cd /opt && git clone -q  https://github.com/confluentinc/kibosh.git &&
cd "/opt/kibosh" && git reset --hard $KIBOSH_VERSION && mkdir "/opt/kibosh/build"
&& cd "/opt/kibosh/build" && ../configure && make -j 2
 
 # Set up the ducker user.
-RUN useradd -ms /bin/bash ducker && mkdir -p /home/ducker/ && rsync -aiq
/root/.ssh/ /home/ducker/.ssh && chown -R ducker /home/ducker/ /mnt/ && echo
'ducker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
+RUN useradd -ms /bin/bash ducker && mkdir -p /home/ducker/ && rsync -aiq
/root/.ssh/ /home/ducker/.ssh && chown -R ducker /home/ducker/ /mnt/ /var/log/ &&
echo 'ducker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
 USER ducker
 
 CMD sudo service ssh start && tail -f /dev/null
diff --git a/tests/docker/ducker-ak b/tests/docker/ducker-ak
index ba8ccf4..a54bd30 100755
--- a/tests/docker/ducker-ak
+++ b/tests/docker/ducker-ak
@@ -41,6 +41,9 @@ docker_run_memory_limit="2000m"
 # The default number of cluster nodes to bring up if a number is not specified.
 default_num_nodes=14
 
+# The default OpenJDK base image.
+default_jdk="openjdk:8"
+
 # The default ducker-ak image name.
 default_image_name="ducker-ak"
 
@@ -215,7 +218,7 @@ ducker_build() {
 
     must_pushd "${ducker_dir}"
     must_do -v -o docker build --memory="${docker_build_memory_limit}" \
-        --build-arg "ducker_creator=${user_name}" -t "${image_name}" \
+        --build-arg "ducker_creator=${user_name}" --build-arg "jdk_version=${jdk_version}"
-t "${image_name}" \
         -f "${ducker_dir}/Dockerfile" ${docker_args} -- .
     docker_status=$?
     must_popd
@@ -264,11 +267,13 @@ ducker_up() {
             -C|--custom-ducktape) set_once custom_ducktape "${2}" "the custom ducktape directory";
shift 2;;
             -f|--force) force=1; shift;;
             -n|--num-nodes) set_once num_nodes "${2}" "number of nodes"; shift 2;;
+            -j|--jdk) set_once jdk_version "${2}" "the OpenJDK base image"; shift 2;;
             *) set_once image_name "${1}" "docker image name"; shift;;
         esac
     done
     [[ -n "${num_nodes}" ]] || num_nodes="${default_num_nodes}"
-    [[ -n "${image_name}" ]] || image_name=ducker-ak
+    [[ -n "${jdk_version}" ]] || jdk_version="${default_jdk}"
+    [[ -n "${image_name}" ]] || image_name="${default_image_name}-${jdk_version/:/-}"
     [[ "${num_nodes}" =~ ^-?[0-9]+$ ]] || \
         die "ducker_up: the number of nodes must be an integer."
     [[ "${num_nodes}" -gt 0 ]] || die "ducker_up: the number of nodes must be greater than
0."


Mime
View raw message