From commits-return-6947-archive-asf-public=cust-asf.ponee.io@kudu.apache.org Wed Feb 6 05:26:10 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id B6C501807A5 for ; Wed, 6 Feb 2019 06:26:09 +0100 (CET) Received: (qmail 43002 invoked by uid 500); 6 Feb 2019 05:26:08 -0000 Mailing-List: contact commits-help@kudu.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kudu.apache.org Delivered-To: mailing list commits@kudu.apache.org Received: (qmail 42986 invoked by uid 99); 6 Feb 2019 05:26:08 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 06 Feb 2019 05:26:08 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 7CB008820A; Wed, 6 Feb 2019 05:26:07 +0000 (UTC) Date: Wed, 06 Feb 2019 05:26:12 +0000 To: "commits@kudu.apache.org" Subject: [kudu] 05/06: KUDU-2411: Set SASL_PATH if needed when starting MiniCluster MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit From: awong@apache.org In-Reply-To: <154943076703.23102.7956893345751034581@gitbox.apache.org> References: <154943076703.23102.7956893345751034581@gitbox.apache.org> X-Git-Host: gitbox.apache.org X-Git-Repo: kudu X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Rev: 0c02110fd2f1c7ca0eef127ba6c3ac1b051f07ab X-Git-NotificationType: diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated Message-Id: <20190206052607.7CB008820A@gitbox.apache.org> This is an automated email from the ASF dual-hosted git repository. awong pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git commit 0c02110fd2f1c7ca0eef127ba6c3ac1b051f07ab Author: Mike Percy AuthorDate: Mon Feb 4 02:54:12 2019 -0800 KUDU-2411: Set SASL_PATH if needed when starting MiniCluster When using the binary artifact on Linux, the MiniCluster should check to see if the sasl2 libraries are included in the artifact and if should set the SASL_PATH environment variable to the location of the sasl2 modules when starting the MiniCluster. This is required because the sasl2 modules are dynamically loaded at runtime. Some refactoring was required to provide the relevant information to set the SASL_PATH when starting the mini cluster process. Additionally, this patch includes some miscellaneous cleanup. Change-Id: Iaaeb30781f4483910c35a20c6d7c76f7f85aa4ce Reviewed-on: http://gerrit.cloudera.org:8080/12351 Reviewed-by: Grant Henke Tested-by: Kudu Jenkins --- .../apache/kudu/test/cluster/KuduBinaryInfo.java | 50 +++++++++++++++++++ .../kudu/test/cluster/KuduBinaryJarExtractor.java | 41 +++++++++++----- .../kudu/test/cluster/KuduBinaryLocator.java | 56 +++++++++++++++++----- .../apache/kudu/test/cluster/MiniKuduCluster.java | 12 +++-- .../test/cluster/TestKuduBinaryJarExtractor.java | 10 ++-- 5 files changed, 133 insertions(+), 36 deletions(-) diff --git a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryInfo.java b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryInfo.java new file mode 100644 index 0000000..a8f3d9a --- /dev/null +++ b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryInfo.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.kudu.test.cluster; + +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.yetus.audience.InterfaceStability; + +/** + * Simple struct to provide various properties of a binary artifact to callers. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class KuduBinaryInfo { + private final String binDir; + private final String saslDir; + + public KuduBinaryInfo(String binDir, String saslDir) { + this.binDir = binDir; + this.saslDir = saslDir; + } + + public KuduBinaryInfo(String binDir) { + this(binDir, null); + } + + /** + * Return the binary directory of an extracted artifact. + */ + public String getBinDir() { return binDir; } + + /** + * Return the SASL module directory of an extracted artifact. + * May be {@code null} if unknown. + */ + public String getSaslDir() { return saslDir; } +} diff --git a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryJarExtractor.java b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryJarExtractor.java index 451abb7..2ddb215 100644 --- a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryJarExtractor.java +++ b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryJarExtractor.java @@ -53,8 +53,6 @@ public class KuduBinaryJarExtractor { "META-INF/apache-kudu-test-binary.properties"; private static final OsDetector DETECTOR = new OsDetector(); - public KuduBinaryJarExtractor() {} - /** Return the thread context classloader or the parent classloader for this class. */ private static ClassLoader getCurrentClassLoader() { ClassLoader loader = Thread.currentThread().getContextClassLoader(); @@ -109,35 +107,52 @@ public class KuduBinaryJarExtractor { * {@link #isKuduBinaryJarOnClasspath()} should return {@code true} before this method is invoked. * * @param destDir path to a destination - * @return the absolute path to the directory containing extracted executables, - * eg. "/tmp/apache-kudu-1.9.0/bin" + * @return information about the extracted artifact * @throws FileNotFoundException if the binary JAR cannot not be located * @throws IOException if the JAR extraction process fails */ - public String extractKuduBinary(String destDir) throws IOException { + public KuduBinaryInfo extractKuduBinaryArtifact(String destDir) throws IOException { Properties binaryProps = getBinaryProps(); if (binaryProps == null) { throw new FileNotFoundException("Could not locate the Kudu binary test jar"); } + String prefix = binaryProps.getProperty("artifact.prefix"); + URL artifactPrefix = getCurrentClassLoader().getResource(prefix); + if (artifactPrefix == null) { + throw new FileNotFoundException("Cannot find Kudu artifact prefix dir: " + prefix); + } + try { - String prefix = binaryProps.getProperty("artifact.prefix"); - URL kuduBinDir = getCurrentClassLoader().getResource(prefix); - if (null == kuduBinDir) { - throw new FileNotFoundException("Cannot find Kudu binary dir: " + prefix); + Path artifactRoot = extractJar(artifactPrefix.toURI(), prefix, Paths.get(destDir)); + Path binDir = Paths.get(artifactRoot.toString(), "bin"); + if (!binDir.toFile().exists()) { + throw new FileNotFoundException("Cannot find Kudu artifact bin dir: " + binDir.toString()); + } + + // Only set the saslDir property if we find it in the artifact, since that affects whether + // the caller needs to set SASL_PATH when executing the binaries. + Path saslDir = Paths.get(artifactRoot.toString(), "lib", "sasl2"); + String saslDirString = null; + if (saslDir.toFile().exists()) { + saslDirString = saslDir.toAbsolutePath().toString(); } - final Path target = Paths.get(destDir); - return extractJar(kuduBinDir.toURI(), target, prefix).toString(); + return new KuduBinaryInfo(binDir.toString(), saslDirString); } catch (URISyntaxException e) { throw new IOException("Cannot unpack Kudu binary jar", e); } } /** + * Extracts the given prefix of the given jar into the target directory. * Accessible for testing only. + * @param src URI of the source jar + * @param prefix prefix of the jar to extract into the destination directory + * @param target destination directory + * @return an absolute path to the extracted artifact, including the prefix portion */ - static Path extractJar(URI src, final Path target, String prefix) throws IOException { + static Path extractJar(URI src, String prefix, final Path target) throws IOException { Preconditions.checkArgument("jar".equals(src.getScheme()), "src URI must use a 'jar' scheme"); if (Files.notExists(target)) { Files.createDirectory(target); @@ -172,6 +187,6 @@ public class KuduBinaryJarExtractor { } }); } - return Paths.get(target.toString(), prefix, "bin"); + return Paths.get(target.toString(), prefix).toAbsolutePath(); } } diff --git a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryLocator.java b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryLocator.java index d1102a2..6a983b8 100644 --- a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryLocator.java +++ b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryLocator.java @@ -17,6 +17,7 @@ package org.apache.kudu.test.cluster; +import com.google.common.base.Preconditions; import com.google.common.io.CharStreams; import org.apache.kudu.test.TempDirUtils; import org.apache.yetus.audience.InterfaceAudience; @@ -29,15 +30,38 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; +import java.util.HashMap; +import java.util.Map; import static java.nio.charset.StandardCharsets.UTF_8; @InterfaceAudience.Private @InterfaceStability.Unstable public class KuduBinaryLocator { - private static final Logger LOG = LoggerFactory.getLogger(KuduBinaryLocator.class); + private static final String SASL_PATH_NAME = "SASL_PATH"; private static final String KUDU_BIN_DIR_PROP = "kuduBinDir"; + private static final Logger LOG = LoggerFactory.getLogger(KuduBinaryLocator.class); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static class ExecutableInfo { + private final String exePath; + private final Map env; + + public ExecutableInfo(String exePath, Map env) { + Preconditions.checkNotNull(exePath); + Preconditions.checkNotNull(env); + this.exePath = exePath; + this.env = env; + } + + /** Path to the executable. */ + public String exePath() { return exePath; } + + /** Any environment variables that should be set when running the executable. */ + public Map environment() { return env; } + } /** * Find the binary directory within the build tree. @@ -47,13 +71,13 @@ public class KuduBinaryLocator { * - If the `kudu` binary is found on the PATH using `which kudu`, * use its parent directory. */ - private static String findBinaryDir() { + private static KuduBinaryInfo findBinaryLocation() { // If kuduBinDir system property is set, use that. String kuduBinDirProp = System.getProperty(KUDU_BIN_DIR_PROP); if (kuduBinDirProp != null) { LOG.info("Using Kudu binary directory specified by system property '{}': {}", KUDU_BIN_DIR_PROP, kuduBinDirProp); - return kuduBinDirProp; + return new KuduBinaryInfo(kuduBinDirProp); } try { @@ -61,7 +85,7 @@ public class KuduBinaryLocator { if (extractor.isKuduBinaryJarOnClasspath()) { String testTmpDir = TempDirUtils.getTempDirectory("kudu-binary-jar").toString(); LOG.info("Using Kudu binary jar directory: {}", testTmpDir); - return extractor.extractKuduBinary(testTmpDir); + return extractor.extractKuduBinaryArtifact(testTmpDir); } } catch (IOException ex) { LOG.warn("Unable to extract a Kudu binary jar", ex); @@ -77,7 +101,7 @@ public class KuduBinaryLocator { String kuduBinary = CharStreams.toString(reader); String kuduBinDir = new File(kuduBinary).getParent(); LOG.info("Using Kudu binary directory found on path with 'which kudu': {}", kuduBinDir); - return kuduBinDir; + return new KuduBinaryInfo(kuduBinDir); } } } catch (IOException | InterruptedException ex) { @@ -89,18 +113,24 @@ public class KuduBinaryLocator { } /** - * @param binName the binary to look for (eg 'kudu-tserver') + * @param exeName the binary to look for (eg 'kudu-tserver') * @return the absolute path of that binary * @throws FileNotFoundException if no such binary is found */ - public static String findBinary(String binName) throws FileNotFoundException { - String binDir = findBinaryDir(); + public static ExecutableInfo findBinary(String exeName) throws FileNotFoundException { + KuduBinaryInfo artifactInfo = findBinaryLocation(); - File candidate = new File(binDir, binName); - if (candidate.canExecute()) { - return candidate.getAbsolutePath(); + File executable = new File(artifactInfo.getBinDir(), exeName); + if (!executable.exists() || !executable.canExecute()) { + throw new FileNotFoundException("Cannot find executable " + exeName + + " in binary directory " + artifactInfo.getBinDir()); } - throw new FileNotFoundException("Cannot find binary " + binName + - " in binary directory " + binDir); + + Map env = new HashMap<>(); + if (artifactInfo.getSaslDir() != null) { + env.put(SASL_PATH_NAME, artifactInfo.getSaslDir()); + } + + return new ExecutableInfo(executable.getAbsolutePath(), env); } } diff --git a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/MiniKuduCluster.java b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/MiniKuduCluster.java index 5dd7b12..2a0fe30 100644 --- a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/MiniKuduCluster.java +++ b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/MiniKuduCluster.java @@ -179,13 +179,15 @@ public class MiniKuduCluster implements AutoCloseable { Preconditions.checkArgument(numMasters > 0, "Need at least one master"); // Start the control shell and the communication channel to it. - List commandLine = Lists.newArrayList( - KuduBinaryLocator.findBinary("kudu"), - "test", - "mini_cluster", - "--serialization=pb"); + KuduBinaryLocator.ExecutableInfo exeInfo = KuduBinaryLocator.findBinary("kudu"); + List commandLine = Lists.newArrayList(exeInfo.exePath(), + "test", + "mini_cluster", + "--serialization=pb"); LOG.info("Starting process: {}", commandLine); ProcessBuilder processBuilder = new ProcessBuilder(commandLine); + processBuilder.environment().putAll(exeInfo.environment()); + miniCluster = processBuilder.start(); miniClusterStdin = new DataOutputStream(miniCluster.getOutputStream()); miniClusterStdout = new DataInputStream(miniCluster.getInputStream()); diff --git a/java/kudu-test-utils/src/test/java/org/apache/kudu/test/cluster/TestKuduBinaryJarExtractor.java b/java/kudu-test-utils/src/test/java/org/apache/kudu/test/cluster/TestKuduBinaryJarExtractor.java index 3bf1d8b..0b6bdda 100644 --- a/java/kudu-test-utils/src/test/java/org/apache/kudu/test/cluster/TestKuduBinaryJarExtractor.java +++ b/java/kudu-test-utils/src/test/java/org/apache/kudu/test/cluster/TestKuduBinaryJarExtractor.java @@ -129,11 +129,11 @@ public class TestKuduBinaryJarExtractor { URI binaryJar = createKuduBinaryJar("osx").toUri(); binaryJar = URI.create("jar:" + binaryJar.toString()); - Path extractedBinDir = - KuduBinaryJarExtractor.extractJar(binaryJar, - Files.createTempDirectory("kudu-test"), - "apache-kudu-1.9.0-SNAPSHOT"); - assertNotNull(extractedBinDir); + Path extractedDir = KuduBinaryJarExtractor.extractJar(binaryJar, + "apache-kudu-1.9.0-SNAPSHOT", + Files.createTempDirectory("kudu-test")); + Path extractedBinDir = Paths.get(extractedDir.toString(), "bin"); + assertTrue(extractedBinDir.toFile().exists()); Path kuduTserver = Paths.get(extractedBinDir.toString(), "kudu-tserver"); assertTrue(Files.exists(kuduTserver));