kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aw...@apache.org
Subject [kudu] 05/06: KUDU-2411: Set SASL_PATH if needed when starting MiniCluster
Date Wed, 06 Feb 2019 05:26:12 GMT
This is an automated email from the ASF dual-hosted git repository.

awong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git

commit 0c02110fd2f1c7ca0eef127ba6c3ac1b051f07ab
Author: Mike Percy <mpercy@apache.org>
AuthorDate: Mon Feb 4 02:54:12 2019 -0800

    KUDU-2411: Set SASL_PATH if needed when starting MiniCluster
    
    When using the binary artifact on Linux, the MiniCluster should check to
    see if the sasl2 libraries are included in the artifact and if should
    set the SASL_PATH environment variable to the location of the
    sasl2 modules when starting the MiniCluster. This is required because
    the sasl2 modules are dynamically loaded at runtime.
    
    Some refactoring was required to provide the relevant information to set
    the SASL_PATH when starting the mini cluster process.
    
    Additionally, this patch includes some miscellaneous cleanup.
    
    Change-Id: Iaaeb30781f4483910c35a20c6d7c76f7f85aa4ce
    Reviewed-on: http://gerrit.cloudera.org:8080/12351
    Reviewed-by: Grant Henke <granthenke@apache.org>
    Tested-by: Kudu Jenkins
---
 .../apache/kudu/test/cluster/KuduBinaryInfo.java   | 50 +++++++++++++++++++
 .../kudu/test/cluster/KuduBinaryJarExtractor.java  | 41 +++++++++++-----
 .../kudu/test/cluster/KuduBinaryLocator.java       | 56 +++++++++++++++++-----
 .../apache/kudu/test/cluster/MiniKuduCluster.java  | 12 +++--
 .../test/cluster/TestKuduBinaryJarExtractor.java   | 10 ++--
 5 files changed, 133 insertions(+), 36 deletions(-)

diff --git a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryInfo.java
b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryInfo.java
new file mode 100644
index 0000000..a8f3d9a
--- /dev/null
+++ b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryInfo.java
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.kudu.test.cluster;
+
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+
+/**
+ * Simple struct to provide various properties of a binary artifact to callers.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class KuduBinaryInfo {
+  private final String binDir;
+  private final String saslDir;
+
+  public KuduBinaryInfo(String binDir, String saslDir) {
+    this.binDir = binDir;
+    this.saslDir = saslDir;
+  }
+
+  public KuduBinaryInfo(String binDir) {
+    this(binDir, null);
+  }
+
+  /**
+   * Return the binary directory of an extracted artifact.
+   */
+  public String getBinDir() { return binDir; }
+
+  /**
+   * Return the SASL module directory of an extracted artifact.
+   * May be {@code null} if unknown.
+   */
+  public String getSaslDir() { return saslDir; }
+}
diff --git a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryJarExtractor.java
b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryJarExtractor.java
index 451abb7..2ddb215 100644
--- a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryJarExtractor.java
+++ b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryJarExtractor.java
@@ -53,8 +53,6 @@ public class KuduBinaryJarExtractor {
       "META-INF/apache-kudu-test-binary.properties";
   private static final OsDetector DETECTOR = new OsDetector();
 
-  public KuduBinaryJarExtractor() {}
-
   /** Return the thread context classloader or the parent classloader for this class. */
   private static ClassLoader getCurrentClassLoader() {
     ClassLoader loader = Thread.currentThread().getContextClassLoader();
@@ -109,35 +107,52 @@ public class KuduBinaryJarExtractor {
    * {@link #isKuduBinaryJarOnClasspath()} should return {@code true} before this method
is invoked.
    *
    * @param destDir path to a destination
-   * @return the absolute path to the directory containing extracted executables,
-   * eg. "/tmp/apache-kudu-1.9.0/bin"
+   * @return information about the extracted artifact
    * @throws FileNotFoundException if the binary JAR cannot not be located
    * @throws IOException if the JAR extraction process fails
    */
-  public String extractKuduBinary(String destDir) throws IOException {
+  public KuduBinaryInfo extractKuduBinaryArtifact(String destDir) throws IOException {
     Properties binaryProps = getBinaryProps();
     if (binaryProps == null) {
       throw new FileNotFoundException("Could not locate the Kudu binary test jar");
     }
 
+    String prefix = binaryProps.getProperty("artifact.prefix");
+    URL artifactPrefix = getCurrentClassLoader().getResource(prefix);
+    if (artifactPrefix == null) {
+      throw new FileNotFoundException("Cannot find Kudu artifact prefix dir: " + prefix);
+    }
+
     try {
-      String prefix = binaryProps.getProperty("artifact.prefix");
-      URL kuduBinDir = getCurrentClassLoader().getResource(prefix);
-      if (null == kuduBinDir) {
-        throw new FileNotFoundException("Cannot find Kudu binary dir: " + prefix);
+      Path artifactRoot = extractJar(artifactPrefix.toURI(), prefix, Paths.get(destDir));
+      Path binDir = Paths.get(artifactRoot.toString(), "bin");
+      if (!binDir.toFile().exists()) {
+        throw new FileNotFoundException("Cannot find Kudu artifact bin dir: " + binDir.toString());
+      }
+
+      // Only set the saslDir property if we find it in the artifact, since that affects
whether
+      // the caller needs to set SASL_PATH when executing the binaries.
+      Path saslDir = Paths.get(artifactRoot.toString(), "lib", "sasl2");
+      String saslDirString = null;
+      if (saslDir.toFile().exists()) {
+        saslDirString = saslDir.toAbsolutePath().toString();
       }
 
-      final Path target = Paths.get(destDir);
-      return extractJar(kuduBinDir.toURI(), target, prefix).toString();
+      return new KuduBinaryInfo(binDir.toString(), saslDirString);
     } catch (URISyntaxException e) {
       throw new IOException("Cannot unpack Kudu binary jar", e);
     }
   }
 
   /**
+   * Extracts the given prefix of the given jar into the target directory.
    * Accessible for testing only.
+   * @param src URI of the source jar
+   * @param prefix prefix of the jar to extract into the destination directory
+   * @param target destination directory
+   * @return an absolute path to the extracted artifact, including the prefix portion
    */
-  static Path extractJar(URI src, final Path target, String prefix) throws IOException {
+  static Path extractJar(URI src, String prefix, final Path target) throws IOException {
     Preconditions.checkArgument("jar".equals(src.getScheme()), "src URI must use a 'jar'
scheme");
     if (Files.notExists(target)) {
       Files.createDirectory(target);
@@ -172,6 +187,6 @@ public class KuduBinaryJarExtractor {
         }
       });
     }
-    return Paths.get(target.toString(), prefix, "bin");
+    return Paths.get(target.toString(), prefix).toAbsolutePath();
   }
 }
diff --git a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryLocator.java
b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryLocator.java
index d1102a2..6a983b8 100644
--- a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryLocator.java
+++ b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryLocator.java
@@ -17,6 +17,7 @@
 
 package org.apache.kudu.test.cluster;
 
+import com.google.common.base.Preconditions;
 import com.google.common.io.CharStreams;
 import org.apache.kudu.test.TempDirUtils;
 import org.apache.yetus.audience.InterfaceAudience;
@@ -29,15 +30,38 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.util.HashMap;
+import java.util.Map;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class KuduBinaryLocator {
-  private static final Logger LOG = LoggerFactory.getLogger(KuduBinaryLocator.class);
 
+  private static final String SASL_PATH_NAME = "SASL_PATH";
   private static final String KUDU_BIN_DIR_PROP = "kuduBinDir";
+  private static final Logger LOG = LoggerFactory.getLogger(KuduBinaryLocator.class);
+
+  @InterfaceAudience.Private
+  @InterfaceStability.Unstable
+  public static class ExecutableInfo {
+    private final String exePath;
+    private final Map<String, String> env;
+
+    public ExecutableInfo(String exePath, Map<String, String> env) {
+      Preconditions.checkNotNull(exePath);
+      Preconditions.checkNotNull(env);
+      this.exePath = exePath;
+      this.env = env;
+    }
+
+    /** Path to the executable. */
+    public String exePath() { return exePath; }
+
+    /** Any environment variables that should be set when running the executable. */
+    public Map<String, String> environment() { return env; }
+  }
 
   /**
    * Find the binary directory within the build tree.
@@ -47,13 +71,13 @@ public class KuduBinaryLocator {
    * - If the `kudu` binary is found on the PATH using `which kudu`,
    * use its parent directory.
    */
-  private static String findBinaryDir() {
+  private static KuduBinaryInfo findBinaryLocation() {
     // If kuduBinDir system property is set, use that.
     String kuduBinDirProp = System.getProperty(KUDU_BIN_DIR_PROP);
     if (kuduBinDirProp != null) {
       LOG.info("Using Kudu binary directory specified by system property '{}': {}",
           KUDU_BIN_DIR_PROP, kuduBinDirProp);
-      return kuduBinDirProp;
+      return new KuduBinaryInfo(kuduBinDirProp);
     }
 
     try {
@@ -61,7 +85,7 @@ public class KuduBinaryLocator {
       if (extractor.isKuduBinaryJarOnClasspath()) {
         String testTmpDir = TempDirUtils.getTempDirectory("kudu-binary-jar").toString();
         LOG.info("Using Kudu binary jar directory: {}", testTmpDir);
-        return extractor.extractKuduBinary(testTmpDir);
+        return extractor.extractKuduBinaryArtifact(testTmpDir);
       }
     } catch (IOException ex) {
       LOG.warn("Unable to extract a Kudu binary jar", ex);
@@ -77,7 +101,7 @@ public class KuduBinaryLocator {
           String kuduBinary = CharStreams.toString(reader);
           String kuduBinDir = new File(kuduBinary).getParent();
           LOG.info("Using Kudu binary directory found on path with 'which kudu': {}", kuduBinDir);
-          return kuduBinDir;
+          return new KuduBinaryInfo(kuduBinDir);
         }
       }
     } catch (IOException | InterruptedException ex) {
@@ -89,18 +113,24 @@ public class KuduBinaryLocator {
   }
 
   /**
-   * @param binName the binary to look for (eg 'kudu-tserver')
+   * @param exeName the binary to look for (eg 'kudu-tserver')
    * @return the absolute path of that binary
    * @throws FileNotFoundException if no such binary is found
    */
-  public static String findBinary(String binName) throws FileNotFoundException {
-    String binDir = findBinaryDir();
+  public static ExecutableInfo findBinary(String exeName) throws FileNotFoundException {
+    KuduBinaryInfo artifactInfo = findBinaryLocation();
 
-    File candidate = new File(binDir, binName);
-    if (candidate.canExecute()) {
-      return candidate.getAbsolutePath();
+    File executable = new File(artifactInfo.getBinDir(), exeName);
+    if (!executable.exists() || !executable.canExecute()) {
+      throw new FileNotFoundException("Cannot find executable " + exeName +
+          " in binary directory " + artifactInfo.getBinDir());
     }
-    throw new FileNotFoundException("Cannot find binary " + binName +
-        " in binary directory " + binDir);
+
+    Map<String, String> env = new HashMap<>();
+    if (artifactInfo.getSaslDir() != null) {
+      env.put(SASL_PATH_NAME, artifactInfo.getSaslDir());
+    }
+
+    return new ExecutableInfo(executable.getAbsolutePath(), env);
   }
 }
diff --git a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/MiniKuduCluster.java
b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/MiniKuduCluster.java
index 5dd7b12..2a0fe30 100644
--- a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/MiniKuduCluster.java
+++ b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/MiniKuduCluster.java
@@ -179,13 +179,15 @@ public class MiniKuduCluster implements AutoCloseable {
     Preconditions.checkArgument(numMasters > 0, "Need at least one master");
 
     // Start the control shell and the communication channel to it.
-    List<String> commandLine = Lists.newArrayList(
-        KuduBinaryLocator.findBinary("kudu"),
-        "test",
-        "mini_cluster",
-        "--serialization=pb");
+    KuduBinaryLocator.ExecutableInfo exeInfo = KuduBinaryLocator.findBinary("kudu");
+    List<String> commandLine = Lists.newArrayList(exeInfo.exePath(),
+                                                  "test",
+                                                  "mini_cluster",
+                                                  "--serialization=pb");
     LOG.info("Starting process: {}", commandLine);
     ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
+    processBuilder.environment().putAll(exeInfo.environment());
+
     miniCluster = processBuilder.start();
     miniClusterStdin = new DataOutputStream(miniCluster.getOutputStream());
     miniClusterStdout = new DataInputStream(miniCluster.getInputStream());
diff --git a/java/kudu-test-utils/src/test/java/org/apache/kudu/test/cluster/TestKuduBinaryJarExtractor.java
b/java/kudu-test-utils/src/test/java/org/apache/kudu/test/cluster/TestKuduBinaryJarExtractor.java
index 3bf1d8b..0b6bdda 100644
--- a/java/kudu-test-utils/src/test/java/org/apache/kudu/test/cluster/TestKuduBinaryJarExtractor.java
+++ b/java/kudu-test-utils/src/test/java/org/apache/kudu/test/cluster/TestKuduBinaryJarExtractor.java
@@ -129,11 +129,11 @@ public class TestKuduBinaryJarExtractor {
     URI binaryJar = createKuduBinaryJar("osx").toUri();
 
     binaryJar = URI.create("jar:" + binaryJar.toString());
-    Path extractedBinDir =
-        KuduBinaryJarExtractor.extractJar(binaryJar,
-            Files.createTempDirectory("kudu-test"),
-            "apache-kudu-1.9.0-SNAPSHOT");
-    assertNotNull(extractedBinDir);
+    Path extractedDir = KuduBinaryJarExtractor.extractJar(binaryJar,
+                                                          "apache-kudu-1.9.0-SNAPSHOT",
+                                                          Files.createTempDirectory("kudu-test"));
+    Path extractedBinDir = Paths.get(extractedDir.toString(), "bin");
+    assertTrue(extractedBinDir.toFile().exists());
 
     Path kuduTserver = Paths.get(extractedBinDir.toString(), "kudu-tserver");
     assertTrue(Files.exists(kuduTserver));


Mime
View raw message