hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vihan...@apache.org
Subject [2/8] hive git commit: Implements the logic to generate and build a docker image in DockerTestPrepPhase
Date Tue, 26 Jun 2018 05:17:38 GMT
Implements the logic to generate and build a docker image in DockerTestPrepPhase


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/41928126
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/41928126
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/41928126

Branch: refs/heads/HIVE-19429
Commit: 41928126427f3b9f0a5732816712041211675ac0
Parents: 9765a43
Author: Vihang Karajgaonkar <vihang@cloudera.com>
Authored: Sat Jun 2 17:36:51 2018 -0700
Committer: Vihang Karajgaonkar <vihang@cloudera.com>
Committed: Mon Jun 4 11:33:21 2018 -0700

----------------------------------------------------------------------
 .../hive/ptest/execution/ContainerClient.java   |  31 ++++
 .../ptest/execution/ContainerClientFactory.java |  58 +++++++
 .../execution/DockerBasedExecutionPhase.java    |   3 +
 .../hive/ptest/execution/HostExecutor.java      |   2 +-
 .../org/apache/hive/ptest/execution/Phase.java  |   1 +
 .../apache/hive/ptest/execution/PrepPhase.java  |  37 +++--
 .../execution/containers/DockerClient.java      |  94 +++++++++++
 .../execution/containers/DockerPrepPhase.java   |  73 +++++++++
 .../src/main/resources/dockerfile-template.vm   |  33 ++++
 .../hive/ptest/execution/AbstractTestPhase.java |   4 +-
 .../ptest/execution/TestPtestOnDockers.java     | 161 +++++++++++++++++++
 .../containers/TestDockerPrepPhase.java         |  73 +++++++++
 12 files changed, 553 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
new file mode 100644
index 0000000..a9a8ead
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
@@ -0,0 +1,31 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution;
+
+import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerClientContext;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+public interface ContainerClient {
+  void defineImage(String dir, String label) throws IOException;
+  String getBuildCommand(String dir, long toWait, TimeUnit unit)
+      throws Exception;
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClientFactory.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClientFactory.java
b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClientFactory.java
new file mode 100644
index 0000000..d14176b
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClientFactory.java
@@ -0,0 +1,58 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.hive.ptest.api.server.TestLogger;
+import org.apache.hive.ptest.execution.containers.DockerClient;
+import org.slf4j.Logger;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public abstract class ContainerClientFactory {
+  public enum ContainerType {
+    DOCKER
+  }
+  public static ContainerClient get(ContainerType containerType, ContainerClientContext context)
throws Exception {
+    switch(containerType) {
+    case DOCKER:
+      return new DockerClient(context);
+    default:
+      throw new Exception("Unknown container type");
+    }
+  }
+
+  public static class ContainerClientContext {
+    final Logger logger;
+    final Map<String, String> templateDefaults;
+    public ContainerClientContext(Logger logger, Map<String, String> templateDefaults)
{
+      this.logger = logger;
+      this.templateDefaults = new HashMap<>(templateDefaults);
+    }
+    public Logger getLogger() {
+      return logger;
+    }
+
+    public Map<String, String> getTemplateDefaults() {
+      return templateDefaults;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java
b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java
index 4b19ccc..d7f14f0 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java
@@ -23,12 +23,15 @@ import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableMap;
 import org.apache.hive.ptest.execution.conf.TestBatch;
 import org.apache.hive.ptest.execution.context.ExecutionContext;
+import org.apache.hive.ptest.execution.ssh.RemoteCommandResult;
 import org.slf4j.Logger;
 
 import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.util.List;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
 public class DockerBasedExecutionPhase extends ExecutionPhase {

http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
index 47347eb..24f0bf8 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
@@ -55,7 +55,7 @@ import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
 
-class HostExecutor {
+public class HostExecutor {
   private final Host mHost;
   private final List<Drone> mDrones;
   private final ListeningExecutorService mExecutor;

http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java
index 34c66ce..45007f0 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java
@@ -160,6 +160,7 @@ public abstract class Phase {
     results.addAll(flatten(futures));
     return results;
   }
+
   private <T extends RemoteCommandResult> List<T> flatten(List<ListenableFuture<List<ListenableFuture<T>>>>
futures)
       throws Exception {
     List<T> results = Lists.newArrayList();

http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
index 8fef413..1f57e3a 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
@@ -24,13 +24,14 @@ import java.io.PrintWriter;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.hive.ptest.execution.ssh.NonZeroExitCodeException;
 import org.slf4j.Logger;
 
 import com.google.common.collect.ImmutableMap;
 
 public class PrepPhase extends Phase {
-  private final File mScratchDir;
-  private final File mPatchFile;
+  protected final File mScratchDir;
+  protected final File mPatchFile;
 
   public PrepPhase(List<HostExecutor> hostExecutors,
       LocalCommandFactory localCommandFactory,
@@ -40,23 +41,12 @@ public class PrepPhase extends Phase {
     this.mScratchDir = scratchDir;
     this.mPatchFile = patchFile;
   }
+
   @Override
   public void execute() throws Exception {
     execLocally("rm -rf $workingDir/scratch");
     execLocally("mkdir -p $workingDir/scratch");
-    if(mPatchFile != null) {
-      File smartApplyPatch = new File(mScratchDir, "smart-apply-patch.sh");
-      PrintWriter writer = new PrintWriter(smartApplyPatch);
-      try {
-        writer.write(Templates.readResource("smart-apply-patch.sh"));
-        if(writer.checkError()) {
-          throw new IOException("Error writing to " + smartApplyPatch);
-        }
-      } finally {
-        writer.close();
-      }
-      execLocally("cp -f " + mPatchFile.getPath() + " $workingDir/scratch/build.patch");
-    }
+    createPatchFiles();
     long start;
     long elapsedTime;
     // source prep
@@ -72,4 +62,21 @@ public class PrepPhase extends Phase {
         TimeUnit.MILLISECONDS);
     logger.info("PERF: source prep took " + elapsedTime + " minutes");
   }
+
+  protected void createPatchFiles()
+      throws IOException, InterruptedException, NonZeroExitCodeException {
+    if(mPatchFile != null) {
+      File smartApplyPatch = new File(mScratchDir, "smart-apply-patch.sh");
+      PrintWriter writer = new PrintWriter(smartApplyPatch);
+      try {
+        writer.write(Templates.readResource("smart-apply-patch.sh"));
+        if(writer.checkError()) {
+          throw new IOException("Error writing to " + smartApplyPatch);
+        }
+      } finally {
+        writer.close();
+      }
+      execLocally("cp -f " + mPatchFile.getPath() + " $workingDir/scratch/build.patch");
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
new file mode 100644
index 0000000..d2428a9
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
@@ -0,0 +1,94 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution.containers;
+
+import org.apache.hive.ptest.api.server.TestLogger;
+import org.apache.hive.ptest.execution.ContainerClient;
+import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerClientContext;
+import org.apache.hive.ptest.execution.Templates;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+public class DockerClient implements ContainerClient {
+
+  private static final String USER = "ptestuser";
+  private static final String HOME_DIR = File.separatorChar + "home" + File.separatorChar
+ USER;
+  private static final String DOCKER_SCRATCH_DIR = HOME_DIR + File.separatorChar + "scratch"
+ File.separatorChar;
+  private static Logger logger;
+  private final ContainerClientContext context;
+  public DockerClient(ContainerClientContext context) {
+    logger = context.getLogger();
+    this.context = context;
+  }
+
+  @Override
+  public void defineImage(String dir, String label) throws IOException {
+    if (label == null)
+      label = UUID.randomUUID().toString();
+    File dockerfile = new File(dir, "Dockerfile");
+    logger.info("Writing {} from template", dockerfile);
+    Map<String, String> templateDefaults = context.getTemplateDefaults();
+    if (!templateDefaults.containsKey("label")) {
+      templateDefaults.put("label", label);
+    } else {
+      //TODO throw?
+    }
+    Templates.writeTemplateResult("dockerfile-template.vm", dockerfile, templateDefaults);
+  }
+
+  @Override
+  public String getBuildCommand(String dir, long toWait, TimeUnit unit) throws Exception
{
+    //TODO timeout not used currently
+    long seconds = TimeUnit.SECONDS.convert(toWait, unit);
+    logger.info("Building image");
+    String dockerBuildCommand =
+        new StringBuilder("docker build")
+            //TODO do we need --tag?
+            //.append(" --tag " + imageName())
+            .append(" --build-arg ")
+            .append(" workingDir=$workingDir")
+            .append(" --build-arg ")
+            .append(" patchFilePath=$workingDir/scratch/build.patch")
+            //TODO confirm if we need antEnvOpts
+            //.append(" antEnvOpts=$antEnvOpts")
+            .append(" --build-arg ")
+            .append(" mavenEnvOpts=$mavenEnvOpts")
+            .append(" --build-arg ")
+            .append(" repository=$repository")
+            .append(" --build-arg ")
+            .append(" branch=$branch")
+            .append(" " + dir)
+            .toString();
+    return dockerBuildCommand;
+  }
+
+  private String imageName() {
+    //TODO add a proper image name using patch number
+    return "Ptest-dummy-build";
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
new file mode 100644
index 0000000..b1d9ad6
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
@@ -0,0 +1,73 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution.containers;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hive.ptest.execution.ContainerClient;
+import org.apache.hive.ptest.execution.ContainerClientFactory;
+import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerClientContext;
+import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerType;
+import org.apache.hive.ptest.execution.HostExecutor;
+import org.apache.hive.ptest.execution.LocalCommandFactory;
+import org.apache.hive.ptest.execution.PrepPhase;
+import org.slf4j.Logger;
+
+import java.io.File;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class DockerPrepPhase extends PrepPhase {
+  private final ContainerClient containerClient;
+  public DockerPrepPhase(List<HostExecutor> hostExecutors, LocalCommandFactory localCommandFactory,
+      ImmutableMap<String, String> templateDefaults, File scratchDir, File patchFile,
+      Logger logger) throws Exception {
+    super(hostExecutors, localCommandFactory, templateDefaults, scratchDir, patchFile, logger);
+    //TODO create context builder and pass the environment variables to the context
+    ContainerClientContext context = new ContainerClientContext(logger, templateDefaults);
+    containerClient = ContainerClientFactory.get(ContainerType.DOCKER, context);
+  }
+
+  @Override
+  public void execute() throws Exception {
+    execLocally("rm -rf $workingDir/scratch");
+    execLocally("mkdir -p $workingDir/scratch");
+    createPatchFiles();
+    long start;
+    long elapsedTime;
+    start = System.currentTimeMillis();
+    //TODO give a proper label to the build
+    containerClient.defineImage(getLocalScratchDir(), null);
+    execLocally(getDockerBuildCommand());
+    elapsedTime = TimeUnit.MINUTES.convert((System.currentTimeMillis() - start),
+        TimeUnit.MILLISECONDS);
+    logger.info("PERF: Docker source prep took " + elapsedTime + " minutes");
+  }
+
+  @VisibleForTesting
+  String getLocalScratchDir() {
+    return mScratchDir.getAbsolutePath();
+  }
+
+  @VisibleForTesting
+  public String getDockerBuildCommand() throws Exception {
+    return containerClient.getBuildCommand(getLocalScratchDir(), 30, TimeUnit.MINUTES);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/resources/dockerfile-template.vm
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/dockerfile-template.vm b/testutils/ptest2/src/main/resources/dockerfile-template.vm
new file mode 100644
index 0000000..3652720
--- /dev/null
+++ b/testutils/ptest2/src/main/resources/dockerfile-template.vm
@@ -0,0 +1,33 @@
+FROM centos
+
+RUN yum upgrade -y && \
+    yum update -y && \
+    yum install -y java-1.8.0-openjdk-devel unzip git maven
+
+RUN useradd -m ptestuser
+RUN mkdir -p /home/ptestuser
+COPY smart-apply-patch.sh scratch/build.patch /home/ptestuser/scratch/
+RUN chown -R ptestuser /home/ptestuser
+RUN chmod -R 755 /home/ptestuser
+RUN ls -l /home
+USER ptestuser
+RUN cd /home/ptestuser
+WORKDIR /home/ptestuser
+ARG workingDir
+ARG mavenEnvOpts
+ARG repository
+ARG branch
+ARG label
+
+RUN export MAVEN_OPTS="$mavenEnvOpts"
+RUN /usr/bin/git clone $repository
+RUN cd hive
+WORKDIR /home/ptestuser/hive
+RUN /usr/bin/git checkout $branch || checkout -b $branch origin/$branch
+RUN /usr/bin/git reset --hard origin/$branch
+RUN /usr/bin/git merge --ff-only origin/$branch
+RUN /home/ptestuser/scratch/smart-apply-patch.sh /home/ptestuser/scratch/build.patch
+RUN /usr/bin/mvn -B -T 4 -q install -Dtest=TestMetastoreConf
+RUN cd itests
+RUN /usr/bin/mvn -B -T 4 -q  install -DskipSparkTests -DskipTests
+RUN echo This build is labeled $label

http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java
b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java
index 71e1a64..01c65a6 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java
@@ -58,6 +58,7 @@ public abstract class AbstractTestPhase {
   protected static final String BRANCH = "branch";
   protected static final String REPOSITORY = "repository";
   protected static final String REPOSITORY_NAME = "repositoryName";
+  protected static final String MAVEN_OPTS = "-Xmx128m";
 
   protected Host host;
   protected File baseDir;
@@ -100,6 +101,7 @@ public abstract class AbstractTestPhase {
         .put("logDir", logDir.getAbsolutePath())
         .put("repository", REPOSITORY)
         .put("repositoryName", REPOSITORY_NAME)
+        .put("mavenEnvOpts", MAVEN_OPTS)
         .build();
     host = new Host(HOST, USER, new String[] { LOCAL_DIR }, 2);
   }
@@ -115,7 +117,7 @@ public abstract class AbstractTestPhase {
     String osName = System.getProperty("os.name");
     return osName.contains("OS X");
   }
-  static File createBaseDir(String name) throws IOException {
+  public static File createBaseDir(String name) throws IOException {
     File baseDir;
     if(isOSX()) {
       // else osx gives ugly temp path which screws up approvals

http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
new file mode 100644
index 0000000..516a12e
--- /dev/null
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
@@ -0,0 +1,161 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import org.apache.commons.io.FileUtils;
+import org.apache.hive.ptest.execution.LocalCommand.CollectLogPolicy;
+import org.apache.hive.ptest.execution.conf.Host;
+import org.apache.hive.ptest.execution.containers.DockerPrepPhase;
+import org.apache.hive.ptest.execution.containers.TestDockerPrepPhase;
+import org.apache.hive.ptest.execution.context.ExecutionContext;
+import org.apache.hive.ptest.execution.ssh.NonZeroExitCodeException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.concurrent.Executors;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+
+/**
+ * This test assumes that the host where this test is being run has docker installed
+ */
+public class TestPtestOnDockers {
+  //TODO add logic to ignore this test if docker is not found on the machine
+
+  private DockerPrepPhase phase;
+  private static File dummyPatchFile;
+  private static final Logger logger = LoggerFactory.getLogger(TestDockerPrepPhase.class);
+
+  private File baseDir;
+  private File scratchDir;
+  private File logDir;
+  private File succeededLogDir;
+  private File failedLogDir;
+  private ListeningExecutorService executor;
+  private LocalCommandFactory localCommandFactory;
+  private LocalCommand localCommand;
+  private ImmutableMap<String, String> templateDefaults;
+  private ImmutableList<HostExecutor> hostExecutors;
+  private HostExecutor hostExecutor;
+  private ExecutionContext executionContext;
+  private HostExecutorBuilder hostExecutorBuilder;
+  private Host host;
+
+  private static final String LOCAL_DIR = "/some/local/dir";
+  private static final String PRIVATE_KEY = "some.private.key";
+  private static final String USER = "someuser";
+  private static final String HOST = "somehost";
+  private static final int INSTANCE = 13;
+  private static final String INSTANCE_NAME = HOST + "-" + USER + "-" + INSTANCE;
+  private static final String REAL_BRANCH = "master";
+  private static final String REAL_REPOSITORY = "https://git-wip-us.apache.org/repos/asf/hive.git";
+  private static final String REAL_REPOSITORY_NAME = "apache-hive";
+  private static final String REAL_MAVEN_OPTS = "-Xmx1g";
+  private MockSSHCommandExecutor sshCommandExecutor;
+  private MockRSyncCommandExecutor rsyncCommandExecutor;
+
+  public void initialize(String name) throws Exception {
+    baseDir = AbstractTestPhase.createBaseDir(name);
+    logDir = Dirs.create(new File(baseDir, "logs"));
+    scratchDir = Dirs.create(new File(baseDir, "scratch"));
+    succeededLogDir = Dirs.create(new File(logDir, "succeeded"));
+    failedLogDir = Dirs.create(new File(logDir, "failed"));
+    executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
+    executionContext = mock(ExecutionContext.class);
+    hostExecutorBuilder = mock(HostExecutorBuilder.class);
+    //use real localCommandFactory
+    localCommandFactory = new LocalCommandFactory(logger);
+    sshCommandExecutor = spy(new MockSSHCommandExecutor(logger));
+    rsyncCommandExecutor = spy(new MockRSyncCommandExecutor(logger));
+    templateDefaults = ImmutableMap.<String, String>builder()
+        .put("localDir", LOCAL_DIR)
+        //use baseDir as working directory
+        .put("workingDir", baseDir.getAbsolutePath())
+        .put("instanceName", INSTANCE_NAME)
+        .put("branch", REAL_BRANCH)
+        .put("logDir", logDir.getAbsolutePath())
+        .put("repository", REAL_REPOSITORY)
+        .put("repositoryName", REAL_REPOSITORY_NAME)
+        .put("mavenEnvOpts", REAL_MAVEN_OPTS)
+        .build();
+    host = new Host(HOST, USER, new String[] { LOCAL_DIR }, 2);
+  }
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    dummyPatchFile = File.createTempFile("dummy", "patch");
+    dummyPatchFile.deleteOnExit();
+    Assert.assertTrue("Could not create dummy patch file " + dummyPatchFile.getAbsolutePath(),
+        dummyPatchFile.exists());
+  }
+
+  @Before
+  public void setup() throws Exception {
+    initialize(getClass().getSimpleName());
+    createHostExecutor();
+    phase = new DockerPrepPhase(hostExecutors, localCommandFactory,
+        templateDefaults, baseDir, dummyPatchFile, logger);
+  }
+
+  private void createHostExecutor() {
+    hostExecutor = new HostExecutor(host, PRIVATE_KEY, executor, sshCommandExecutor,
+        rsyncCommandExecutor, templateDefaults, scratchDir, succeededLogDir, failedLogDir,
1, true,
+        logger);
+    hostExecutors = ImmutableList.of(hostExecutor);
+  }
+
+  @After
+  public void teardown() {
+    phase = null;
+    FileUtils.deleteQuietly(baseDir);
+  }
+
+  /**
+   * This test requires docker to be installed to test on local machine
+   * @throws Exception
+   */
+  @Test
+  public void testDockerFile() throws Exception {
+    phase.execute();
+    File dockerFile = new File(scratchDir, "Dockerfile");
+    Assert.assertTrue("Docker file was not found" , dockerFile.exists());
+    Assert.assertTrue("patch file not found", new File(scratchDir, "build.patch").exists());
+    LocalCommandFactory localCommandFactory = new LocalCommandFactory(logger);
+    CollectLogPolicy localCollector = new CollectLogPolicy(logger);
+    localCommandFactory.create(localCollector, "cp -f " + dockerFile.getAbsolutePath() +
" /tmp/myDockerFile");
+    LocalCommand localCmd = localCommandFactory.create(localCollector, phase.getDockerBuildCommand());
+    if(localCmd.getExitCode() != 0) {
+      throw new NonZeroExitCodeException(String.format(
+          "Command '%s' failed with exit status %d and output '%s'",
+          localCmd, localCmd.getExitCode(), localCollector.getOutput()));
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/containers/TestDockerPrepPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/containers/TestDockerPrepPhase.java
b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/containers/TestDockerPrepPhase.java
new file mode 100644
index 0000000..8efc5e0
--- /dev/null
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/containers/TestDockerPrepPhase.java
@@ -0,0 +1,73 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution.containers;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hive.ptest.execution.AbstractTestPhase;
+
+import org.apache.hive.ptest.execution.LocalCommand;
+import org.apache.hive.ptest.execution.LocalCommand.CollectLogPolicy;
+import org.apache.hive.ptest.execution.LocalCommandFactory;
+import org.apache.hive.ptest.execution.ssh.NonZeroExitCodeException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+public class TestDockerPrepPhase extends AbstractTestPhase {
+  private DockerPrepPhase phase;
+  private static File dummyPatchFile;
+  private static final Logger logger = LoggerFactory.getLogger(TestDockerPrepPhase.class);
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    dummyPatchFile = File.createTempFile("dummy", "patch");
+    dummyPatchFile.deleteOnExit();
+    Assert.assertTrue("Could not create dummy patch file " + dummyPatchFile.getAbsolutePath(),
+        dummyPatchFile.exists());
+  }
+
+  @Before
+  public void setup() throws Exception {
+    initialize(getClass().getSimpleName());
+    createHostExecutor();
+    phase = new DockerPrepPhase(hostExecutors, localCommandFactory,
+        templateDefaults, baseDir, dummyPatchFile, logger);
+  }
+
+  @After
+  public void teardown() {
+    phase = null;
+    FileUtils.deleteQuietly(baseDir);
+  }
+
+  @Test
+  public void testExecute() throws Exception {
+    phase.execute();
+    Assert.assertNotNull("Scratch directory needs to be set", phase.getLocalScratchDir());
+    File dockerFile = new File(phase.getLocalScratchDir(), "Dockerfile");
+    Assert.assertTrue("Docker file not found", dockerFile.exists());
+  }
+}


Mime
View raw message