mesos-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vinodk...@apache.org
Subject [1/2] git commit: Fixed fetcher to properly fetch multiple URIs.
Date Wed, 12 Mar 2014 19:16:53 GMT
Repository: mesos
Updated Branches:
  refs/heads/master ffda972f7 -> 8cf49ee7d


Fixed fetcher to properly fetch multiple URIs.

Review: https://reviews.apache.org/r/18974


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/d71b9455
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/d71b9455
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/d71b9455

Branch: refs/heads/master
Commit: d71b94556f7f50cca9be2b43ae4a7c4a642a0435
Parents: ffda972
Author: Dominic Hamon <dhamon@twopensource.com>
Authored: Wed Mar 12 12:16:00 2014 -0700
Committer: Vinod Kone <vinod@twitter.com>
Committed: Wed Mar 12 12:16:00 2014 -0700

----------------------------------------------------------------------
 src/Makefile.am                                 |   1 +
 src/slave/containerizer/mesos_containerizer.cpp |  59 +++---
 src/tests/containerizer_tests.cpp               | 184 +++++++++++++++++++
 3 files changed, 223 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/d71b9455/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index c18b311..ce7913b 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -862,6 +862,7 @@ mesos_tests_SOURCES =				\
   tests/attributes_tests.cpp			\
   tests/authentication_tests.cpp		\
   tests/containerizer.cpp			\
+  tests/containerizer_tests.cpp			\
   tests/environment.cpp				\
   tests/examples_tests.cpp			\
   tests/exception_tests.cpp			\

http://git-wip-us.apache.org/repos/asf/mesos/blob/d71b9455/src/slave/containerizer/mesos_containerizer.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos_containerizer.cpp b/src/slave/containerizer/mesos_containerizer.cpp
index 9bf9829..c1a3e0f 100644
--- a/src/slave/containerizer/mesos_containerizer.cpp
+++ b/src/slave/containerizer/mesos_containerizer.cpp
@@ -53,6 +53,43 @@ using state::RunState;
 Future<Nothing> _nothing() { return Nothing(); }
 
 
+// Helper method to build the command sent to the fetcher.
+std::string buildCommand(
+    const CommandInfo& commandInfo,
+    const std::string& directory,
+    const Option<std::string>& user,
+    const Flags& flags)
+{
+  // Prepare the environment variables to pass to mesos-fetcher.
+  string uris = "";
+  foreach (const CommandInfo::URI& uri, commandInfo.uris()) {
+    uris += uri.value() + "+" +
+            (uri.has_executable() && uri.executable() ? "1" : "0");
+    uris += " ";
+  }
+  // Remove extra space at the end.
+  uris = strings::trim(uris);
+
+  // Use /usr/bin/env to set the environment variables for the fetcher
+  // subprocess because we cannot pollute the slave's environment.
+  // TODO(idownes): Remove this once Subprocess accepts environment variables.
+  string command = "/usr/bin/env";
+  command += " MESOS_EXECUTOR_URIS=\"" + uris + "\"";
+  command += " MESOS_WORK_DIRECTORY=" + directory;
+  if (user.isSome()) {
+    command += " MESOS_USER=" + user.get();
+  }
+  if (!flags.frameworks_home.empty()) {
+    command += " MESOS_FRAMEWORKS_HOME=" + flags.frameworks_home;
+  }
+  if (!flags.hadoop_home.empty()) {
+    command += " HADOOP_HOME=" + flags.hadoop_home;
+  }
+
+  return command;
+}
+
+
 MesosContainerizer::MesosContainerizer(
     const Flags& flags,
     bool local,
@@ -464,27 +501,7 @@ Future<Nothing> MesosContainerizerProcess::fetch(
     return Failure("Could not fetch URIs: failed to find mesos-fetcher");
   }
 
-  // Prepare the environment variables to pass to mesos-fetcher.
-  string uris = "";
-  foreach (const CommandInfo::URI& uri, commandInfo.uris()) {
-    uris += uri.value() + "+" +
-            (uri.has_executable() && uri.executable() ? "1" : "0");
-    uris += " ";
-  }
-  // Remove extra space at the end.
-  uris = strings::trim(uris);
-
-  // Use /usr/bin/env to set the environment variables for the fetcher
-  // subprocess because we cannot pollute the slave's environment.
-  // TODO(idownes): Remove this once Subprocess accepts environment variables.
-  string command = "/usr/bin/env";
-  command += " MESOS_EXECUTOR_URIS=" + uris;
-  command += " MESOS_WORK_DIRECTORY=" + directory;
-  if (user.isSome()) {
-    command += " MESOS_USER=" + user.get();
-  }
-  command += " MESOS_FRAMEWORKS_HOME=" + flags.frameworks_home;
-  command += " HADOOP_HOME=" + flags.hadoop_home;
+  string command = buildCommand(commandInfo, directory, user, flags);
 
   // Now the actual mesos-fetcher command.
   command += " " + realpath.get();

http://git-wip-us.apache.org/repos/asf/mesos/blob/d71b9455/src/tests/containerizer_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer_tests.cpp b/src/tests/containerizer_tests.cpp
new file mode 100644
index 0000000..b539070
--- /dev/null
+++ b/src/tests/containerizer_tests.cpp
@@ -0,0 +1,184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include <mesos/mesos.hpp>
+
+#include <slave/containerizer/mesos_containerizer.hpp>
+#include <slave/flags.hpp>
+
+using namespace mesos;
+using namespace mesos::internal;
+using namespace mesos::internal::slave;
+
+using std::string;
+using std::vector;
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+// Forward declaration.
+string buildCommand(
+    const CommandInfo& commandInfo,
+    const string& directory,
+    const Option<string>& user,
+    const Flags& flags);
+
+}  // namespace slave {
+}  // namespace internal {
+}  // namespace mesos {
+
+class MesosContainerizerProcessTest : public ::testing::Test {};
+
+
+TEST_F(MesosContainerizerProcessTest, Simple) {
+  CommandInfo commandInfo;
+  CommandInfo::URI uri;
+  uri.set_value("hdfs:///uri");
+  uri.set_executable(false);
+  commandInfo.add_uris()->MergeFrom(uri);
+
+  string directory = "/tmp/directory";
+  Option<string> user = "user";
+
+  Flags flags;
+  flags.frameworks_home = "/tmp/frameworks";
+  flags.hadoop_home = "/tmp/hadoop";
+
+  string command = buildCommand(commandInfo, directory, user, flags);
+
+  EXPECT_STREQ(
+      "/usr/bin/env "
+      "MESOS_EXECUTOR_URIS=\"hdfs:///uri+0\" "
+      "MESOS_WORK_DIRECTORY=/tmp/directory "
+      "MESOS_USER=user "
+      "MESOS_FRAMEWORKS_HOME=/tmp/frameworks "
+      "HADOOP_HOME=/tmp/hadoop",
+      command.c_str());
+}
+
+
+TEST_F(MesosContainerizerProcessTest, MultipleURIs) {
+  CommandInfo commandInfo;
+  CommandInfo::URI uri;
+  uri.set_value("hdfs:///uri1");
+  uri.set_executable(false);
+  commandInfo.add_uris()->MergeFrom(uri);
+  uri.set_value("hdfs:///uri2");
+  uri.set_executable(true);
+  commandInfo.add_uris()->MergeFrom(uri);
+
+  string directory = "/tmp/directory";
+  Option<string> user("user");
+
+  Flags flags;
+  flags.frameworks_home = "/tmp/frameworks";
+  flags.hadoop_home = "/tmp/hadoop";
+
+  string command = buildCommand(commandInfo, directory, user, flags);
+
+  EXPECT_STREQ(
+      "/usr/bin/env "
+      "MESOS_EXECUTOR_URIS=\"hdfs:///uri1+0 hdfs:///uri2+1\" "
+      "MESOS_WORK_DIRECTORY=/tmp/directory "
+      "MESOS_USER=user "
+      "MESOS_FRAMEWORKS_HOME=/tmp/frameworks "
+      "HADOOP_HOME=/tmp/hadoop",
+      command.c_str());
+}
+
+
+TEST_F(MesosContainerizerProcessTest, NoUser) {
+  CommandInfo commandInfo;
+  CommandInfo::URI uri;
+  uri.set_value("hdfs:///uri");
+  uri.set_executable(false);
+  commandInfo.add_uris()->MergeFrom(uri);
+
+  string directory = "/tmp/directory";
+
+  Flags flags;
+  flags.frameworks_home = "/tmp/frameworks";
+  flags.hadoop_home = "/tmp/hadoop";
+
+  string command = buildCommand(commandInfo, directory, None(), flags);
+
+  EXPECT_STREQ(
+      "/usr/bin/env "
+      "MESOS_EXECUTOR_URIS=\"hdfs:///uri+0\" "
+      "MESOS_WORK_DIRECTORY=/tmp/directory "
+      "MESOS_FRAMEWORKS_HOME=/tmp/frameworks "
+      "HADOOP_HOME=/tmp/hadoop",
+      command.c_str());
+}
+
+
+TEST_F(MesosContainerizerProcessTest, EmptyHadoop) {
+  CommandInfo commandInfo;
+  CommandInfo::URI uri;
+  uri.set_value("hdfs:///uri");
+  uri.set_executable(false);
+  commandInfo.add_uris()->MergeFrom(uri);
+
+  string directory = "/tmp/directory";
+  Option<string> user = "user";
+
+  Flags flags;
+  flags.frameworks_home = "/tmp/frameworks";
+  flags.hadoop_home = "";
+
+  string command = buildCommand(commandInfo, directory, user, flags);
+
+  EXPECT_STREQ(
+      "/usr/bin/env "
+      "MESOS_EXECUTOR_URIS=\"hdfs:///uri+0\" "
+      "MESOS_WORK_DIRECTORY=/tmp/directory "
+      "MESOS_USER=user "
+      "MESOS_FRAMEWORKS_HOME=/tmp/frameworks",
+      command.c_str());
+}
+
+
+TEST_F(MesosContainerizerProcessTest, NoHadoop) {
+  CommandInfo commandInfo;
+  CommandInfo::URI uri;
+  uri.set_value("hdfs:///uri");
+  uri.set_executable(false);
+  commandInfo.add_uris()->MergeFrom(uri);
+
+  string directory = "/tmp/directory";
+  Option<string> user = "user";
+
+  Flags flags;
+  flags.frameworks_home = "/tmp/frameworks";
+
+  string command = buildCommand(commandInfo, directory, user, flags);
+
+  EXPECT_STREQ(
+      "/usr/bin/env "
+      "MESOS_EXECUTOR_URIS=\"hdfs:///uri+0\" "
+      "MESOS_WORK_DIRECTORY=/tmp/directory "
+      "MESOS_USER=user "
+      "MESOS_FRAMEWORKS_HOME=/tmp/frameworks",
+      command.c_str());
+}


Mime
View raw message