whirr-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From as...@apache.org
Subject svn commit: r1097244 - in /incubator/whirr/trunk: ./ cli/src/main/java/org/apache/whirr/cli/ cli/src/main/java/org/apache/whirr/cli/command/ cli/src/test/java/org/apache/whirr/cli/command/ core/src/main/java/org/apache/whirr/
Date Wed, 27 Apr 2011 21:13:34 GMT
Author: asavu
Date: Wed Apr 27 21:13:34 2011
New Revision: 1097244

URL: http://svn.apache.org/viewvc?rev=1097244&view=rev
Log:
WHIRR-173. Add ClusterAction for generic script execution (asavu)

Added:
    incubator/whirr/trunk/cli/src/main/java/org/apache/whirr/cli/command/RunScriptCommand.java
    incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/RunScriptCommandTest.java
Modified:
    incubator/whirr/trunk/CHANGES.txt
    incubator/whirr/trunk/cli/src/main/java/org/apache/whirr/cli/Main.java
    incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/DestroyInstanceCommandTest.java
    incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ClusterController.java

Modified: incubator/whirr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/CHANGES.txt?rev=1097244&r1=1097243&r2=1097244&view=diff
==============================================================================
--- incubator/whirr/trunk/CHANGES.txt (original)
+++ incubator/whirr/trunk/CHANGES.txt Wed Apr 27 21:13:34 2011
@@ -46,6 +46,8 @@ Trunk (unreleased changes)
     WHIRR-280. Create a blob cache that could be used for storing local 
     files (asavu)
 
+    WHIRR-173. Add ClusterAction for generic script execution (asavu)
+
   BUG FIXES
 
     WHIRR-253. ZooKeeper service should only authorize ingress to ZooKeeper 

Modified: incubator/whirr/trunk/cli/src/main/java/org/apache/whirr/cli/Main.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/cli/src/main/java/org/apache/whirr/cli/Main.java?rev=1097244&r1=1097243&r2=1097244&view=diff
==============================================================================
--- incubator/whirr/trunk/cli/src/main/java/org/apache/whirr/cli/Main.java (original)
+++ incubator/whirr/trunk/cli/src/main/java/org/apache/whirr/cli/Main.java Wed Apr 27 21:13:34
2011
@@ -24,6 +24,7 @@ import org.apache.whirr.cli.command.Dest
 import org.apache.whirr.cli.command.DestroyInstanceCommand;
 import org.apache.whirr.cli.command.LaunchClusterCommand;
 import org.apache.whirr.cli.command.ListClusterCommand;
+import org.apache.whirr.cli.command.RunScriptCommand;
 import org.apache.whirr.cli.command.VersionCommand;
 import org.apache.whirr.service.ClusterActionHandler;
 
@@ -99,7 +100,8 @@ public class Main {
         new LaunchClusterCommand(),
         new DestroyClusterCommand(),
         new DestroyInstanceCommand(),
-        new ListClusterCommand()
+        new ListClusterCommand(),
+        new RunScriptCommand()
     );
     int rc = main.run(System.in, System.out, System.err, Arrays.asList(args));
     System.exit(rc);

Added: incubator/whirr/trunk/cli/src/main/java/org/apache/whirr/cli/command/RunScriptCommand.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/cli/src/main/java/org/apache/whirr/cli/command/RunScriptCommand.java?rev=1097244&view=auto
==============================================================================
--- incubator/whirr/trunk/cli/src/main/java/org/apache/whirr/cli/command/RunScriptCommand.java
(added)
+++ incubator/whirr/trunk/cli/src/main/java/org/apache/whirr/cli/command/RunScriptCommand.java
Wed Apr 27 21:13:34 2011
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.cli.command;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.io.Files;
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.apache.commons.lang.StringUtils;
+import org.apache.whirr.ClusterController;
+import org.apache.whirr.ClusterControllerFactory;
+import org.apache.whirr.ClusterSpec;
+import org.jclouds.compute.domain.ExecResponse;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.scriptbuilder.domain.Statement;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+
+import static org.jclouds.compute.predicates.NodePredicates.withIds;
+import static org.jclouds.scriptbuilder.domain.Statements.exec;
+
+public class RunScriptCommand extends  AbstractClusterSpecCommand {
+
+  private OptionSpec<String> rolesOption = parser
+    .accepts("roles", "List of comma separated role names. " +
+      "E.g. zookeeper,hadoop-namenode")
+    .withRequiredArg()
+    .ofType(String.class);
+
+  private OptionSpec<String> instancesOption = parser
+    .accepts("instances", "List of comma separated instance IDs")
+    .withRequiredArg()
+    .ofType(String.class);
+
+  private OptionSpec<String> scriptOption = parser
+    .accepts("script", "Path to script file to execute.")
+    .withRequiredArg()
+    .ofType(String.class);
+
+  public RunScriptCommand() {
+    this(new ClusterControllerFactory());
+  }
+
+  public RunScriptCommand(ClusterControllerFactory factory) {
+    super("run-script", "Run a script on a specific instance or a " +
+      "group of instances matching a role name", factory);
+  }
+
+  @Override
+  public int run(InputStream in, PrintStream out, PrintStream err,
+                 List<String> args) throws Exception {
+
+    OptionSet optionSet = parser.parse(args.toArray(new String[0]));
+    if (!optionSet.has(scriptOption)) {
+      err.println("Please specify a script file to be executed.");
+      printUsage(parser, err);
+      return -1;
+    }
+
+    if (!(new File(optionSet.valueOf(scriptOption))).exists()) {
+      err.printf("Script file '%s' not found.", optionSet.valueOf(scriptOption));
+      printUsage(parser, err);
+      return -2;
+    }
+
+    try {
+      ClusterSpec spec = getClusterSpec(optionSet);
+      ClusterController controller = createClusterController(spec.getServiceName());
+
+      Predicate<NodeMetadata> condition = buildFilterPredicate(optionSet, spec);
+
+      return handleScriptOutput(out, err, controller.runScriptOnNodesMatching(
+        spec, condition, execFile(optionSet.valueOf(scriptOption))));
+
+    } catch(IllegalArgumentException e) {
+      err.println(e.getMessage());
+      printUsage(parser, err);
+      return -3;
+    }
+  }
+
+  private Predicate<NodeMetadata> buildFilterPredicate(OptionSet optionSet, ClusterSpec
spec)
+      throws IOException {
+
+    Predicate<NodeMetadata> condition = Predicates.alwaysTrue();
+
+    if (optionSet.has(instancesOption)) {
+      String[] ids = optionSet.valueOf(instancesOption).split(",");
+      return Predicates.and(condition, withIds(ids));
+
+    } else if(optionSet.has(rolesOption)) {
+      String[] roles = optionSet.valueOf(rolesOption).split(",");
+      List<String> ids = Lists.newArrayList();
+
+      for(String line : Files.readLines(
+        new File(spec.getClusterDirectory(), "instances"),
+        Charset.defaultCharset())) {
+
+        if (containsAny(line, roles)) {
+         ids.add(line.split("\t")[0].split("\\/")[1]);
+        }
+      }
+
+      condition = Predicates.and(condition,
+        withIds(ids.toArray(new String[0])));
+    }
+    return condition;
+  }
+
+  private int handleScriptOutput(PrintStream out, PrintStream err,
+                                 Map<? extends NodeMetadata, ExecResponse> responses)
{
+    int rc = 0;
+    for (Map.Entry<? extends NodeMetadata, ExecResponse> entry : responses.entrySet())
{
+      out.printf("** Node %s: %s%n", entry.getKey().getId(),
+        Iterables.concat(entry.getKey().getPrivateAddresses(),
+          entry.getKey().getPublicAddresses()));
+
+      ExecResponse response = entry.getValue();
+      if (response.getExitCode() != 0) {
+        rc = response.getExitCode();
+      }
+      out.printf("%s%n", response.getOutput());
+      err.printf("%s%n", response.getError());
+    }
+    return rc;
+  }
+
+  private boolean containsAny(String line, String[] roles) {
+    for (String role : roles) {
+      if (line.contains(role)) return true;
+    }
+    return false;
+  }
+
+  private Statement execFile(String filePath) throws IOException {
+    return exec(getFileContent(filePath));
+  }
+
+  private String getFileContent(String filePath) throws IOException {
+    return StringUtils.join(Files.readLines(new File(filePath),
+        Charset.defaultCharset()),
+      "\n");
+  }
+
+  private void printUsage(OptionParser parser,
+                          PrintStream stream) throws IOException {
+    stream.println("Usage: whirr run-script [OPTIONS] --script <script> " +
+      "[--instances id1,id2] [--roles role1,role2]");
+    stream.println();
+    parser.printHelpOn(stream);
+  }
+
+}

Modified: incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/DestroyInstanceCommandTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/DestroyInstanceCommandTest.java?rev=1097244&r1=1097243&r2=1097244&view=diff
==============================================================================
--- incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/DestroyInstanceCommandTest.java
(original)
+++ incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/DestroyInstanceCommandTest.java
Wed Apr 27 21:13:34 2011
@@ -79,7 +79,7 @@ public class DestroyInstanceCommandTest 
     when(factory.create((String) any())).thenReturn(controller);
 
     DestroyInstanceCommand command = new DestroyInstanceCommand(factory);
-        Map<String, File> keys = KeyPair.generateTemporaryFiles();
+    Map<String, File> keys = KeyPair.generateTemporaryFiles();
 
     int rc = command.run(null, out, null, Lists.newArrayList(
         "--instance-id", "region/instanceid",

Added: incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/RunScriptCommandTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/RunScriptCommandTest.java?rev=1097244&view=auto
==============================================================================
--- incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/RunScriptCommandTest.java
(added)
+++ incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/RunScriptCommandTest.java
Wed Apr 27 21:13:34 2011
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.cli.command;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Lists;
+import com.jcraft.jsch.JSchException;
+import org.apache.whirr.ClusterController;
+import org.apache.whirr.ClusterControllerFactory;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.util.KeyPair;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.PrintStream;
+import java.util.Map;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+import static org.jclouds.compute.predicates.NodePredicates.withIds;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class RunScriptCommandTest {
+
+  private ByteArrayOutputStream outBytes;
+  private PrintStream out;
+
+  private ByteArrayOutputStream errBytes;
+  private PrintStream err;
+
+  @Before
+  public void setUp() {
+    outBytes = new ByteArrayOutputStream();
+    out = new PrintStream(outBytes);
+
+    errBytes = new ByteArrayOutputStream();
+    err = new PrintStream(errBytes);
+  }
+
+  @Test
+  public void testScriptPathIsMandatory() throws Exception {
+    RunScriptCommand command = new RunScriptCommand();
+
+    int rc = command.run(null, out, err, Lists.<String>newArrayList());
+    assertThat(rc, is(-1));
+
+    assertThat(errBytes.toString(),
+      containsString("Please specify a script file to be executed."));
+  }
+
+  @Test
+  public void testRunScriptByInstanceId() throws Exception {
+    ClusterControllerFactory factory = mock(ClusterControllerFactory.class);
+    ClusterController controller = mock(ClusterController.class);
+
+    when(factory.create((String)any())).thenReturn(controller);
+
+    RunScriptCommand command = new RunScriptCommand(factory);
+    Map<String, File> keys = KeyPair.generateTemporaryFiles();
+
+    int rc = command.run(null, out, System.err, Lists.newArrayList(
+        "--script", "/dev/null",
+        "--instances", "A,B",
+        "--cluster-name", "test-cluster",
+        "--provider", "provider",
+        "--identity", "myusername", "--credential", "mypassword",
+        "--private-key-file", keys.get("private").getAbsolutePath()
+      ));
+    assertThat(rc, is(0));
+
+    ArgumentCaptor<Predicate> predicate = ArgumentCaptor.forClass(Predicate.class);
+    verify(controller).runScriptOnNodesMatching(
+      (ClusterSpec)any(), predicate.capture(), (Statement) any());
+
+    // check predicate equality by using the object string representation
+
+    Predicate<NodeMetadata> expected = Predicates.and(
+      Predicates.<NodeMetadata>alwaysTrue(), withIds("A", "B"));
+    assertThat(predicate.getValue().toString(), is(expected.toString()));
+  }
+
+  @Test
+  public void testRunScriptByRole() {
+    // TODO add this test in WHIRR-246 (it should be easy to mock the cluster state)
+  }
+
+}

Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ClusterController.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ClusterController.java?rev=1097244&r1=1097243&r2=1097244&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ClusterController.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ClusterController.java Wed Apr
27 21:13:34 2011
@@ -21,6 +21,7 @@ package org.apache.whirr;
 import com.google.common.base.Charsets;
 import com.google.common.base.Joiner;
 import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
 import com.google.common.io.Files;
 
 import java.io.File;
@@ -35,13 +36,20 @@ import org.apache.whirr.actions.DestroyC
 import org.apache.whirr.service.ClusterActionHandler;
 import org.apache.whirr.service.ComputeServiceContextBuilder;
 import org.jclouds.compute.ComputeService;
+import org.jclouds.compute.ComputeServiceContext;
 import org.jclouds.compute.ComputeServiceContextFactory;
+import org.jclouds.compute.RunScriptOnNodesException;
 import org.jclouds.compute.domain.ComputeMetadata;
+import org.jclouds.compute.domain.ExecResponse;
 import org.jclouds.compute.domain.NodeMetadata;
 import org.jclouds.compute.domain.NodeState;
+import org.jclouds.domain.Credentials;
+import org.jclouds.scriptbuilder.domain.Statement;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.jclouds.compute.options.RunScriptOptions.Builder.overrideCredentialsWith;
+
 /**
  * This class is used to start and stop clusters.
  */
@@ -109,6 +117,7 @@ public class ClusterController {
   
   /**
    * Stop the cluster and destroy all resources associated with it.
+   *
    * @throws IOException if there is a problem while stopping the cluster. The
    * cluster may or may not have been stopped.
    * @throws InterruptedException if the thread is interrupted.
@@ -129,6 +138,20 @@ public class ClusterController {
 
     LOG.info("Instance {} destroyed", instanceId);
   }
+
+  public Map<? extends NodeMetadata, ExecResponse> runScriptOnNodesMatching(ClusterSpec
spec,
+        Predicate<NodeMetadata> condition, Statement statement) throws IOException,
RunScriptOnNodesException {
+
+    Credentials credentials = new Credentials(spec.getClusterUser(), spec.getPrivateKey());
+    ComputeServiceContext context = ComputeServiceContextBuilder.build(spec);
+    try {
+      condition = Predicates.and(runningInGroup(spec.getClusterName()), condition);
+      return context.getComputeService().runScriptOnNodesMatching(condition,
+        statement, overrideCredentialsWith(credentials).wrapInInitScript(false).runAsRoot(false));
+    } finally {
+      context.close();
+    }
+  }
   
   public Set<? extends NodeMetadata> getNodes(ClusterSpec clusterSpec)
     throws IOException, InterruptedException {



Mime
View raw message