mesos-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qianzh...@apache.org
Subject [4/6] mesos git commit: Implemented the conversion from AUFS whiteouts to OverlayFS whiteouts.
Date Sat, 05 Nov 2016 08:40:11 GMT
Implemented the conversion from AUFS whiteouts to OverlayFS whiteouts.

Review: https://reviews.apache.org/r/53161


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3800cfd0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3800cfd0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3800cfd0

Branch: refs/heads/master
Commit: 3800cfd0d2d65e649b21d079e1c1a4ce743558dd
Parents: ac62ee2
Author: Qian Zhang <zhangqxa@cn.ibm.com>
Authored: Tue Oct 25 11:20:26 2016 +0800
Committer: Qian Zhang <zhq527725@gmail.com>
Committed: Sat Nov 5 16:18:30 2016 +0800

----------------------------------------------------------------------
 src/Makefile.am                                 |   2 +
 .../mesos/provisioner/docker/store.cpp          |  24 +++-
 .../containerizer/mesos/provisioner/utils.cpp   | 111 +++++++++++++++++++
 .../containerizer/mesos/provisioner/utils.hpp   |  36 ++++++
 4 files changed, 169 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/3800cfd0/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index f08b986..5a47c93 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -842,6 +842,7 @@ libmesos_no_3rdparty_la_SOURCES +=					\
   slave/containerizer/mesos/provisioner/paths.cpp			\
   slave/containerizer/mesos/provisioner/provisioner.cpp			\
   slave/containerizer/mesos/provisioner/store.cpp			\
+  slave/containerizer/mesos/provisioner/utils.cpp			\
   slave/containerizer/mesos/provisioner/appc/cache.cpp			\
   slave/containerizer/mesos/provisioner/appc/fetcher.cpp		\
   slave/containerizer/mesos/provisioner/appc/paths.cpp			\
@@ -976,6 +977,7 @@ libmesos_no_3rdparty_la_SOURCES +=					\
   slave/containerizer/mesos/provisioner/paths.hpp			\
   slave/containerizer/mesos/provisioner/provisioner.hpp			\
   slave/containerizer/mesos/provisioner/store.hpp			\
+  slave/containerizer/mesos/provisioner/utils.hpp			\
   slave/containerizer/mesos/provisioner/appc/cache.hpp			\
   slave/containerizer/mesos/provisioner/appc/fetcher.hpp		\
   slave/containerizer/mesos/provisioner/appc/paths.hpp			\

http://git-wip-us.apache.org/repos/asf/mesos/blob/3800cfd0/src/slave/containerizer/mesos/provisioner/docker/store.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/store.cpp b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
index e192f86..9dccd06 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
@@ -30,6 +30,9 @@
 
 #include <mesos/docker/spec.hpp>
 
+#include "slave/containerizer/mesos/provisioner/constants.hpp"
+#include "slave/containerizer/mesos/provisioner/utils.hpp"
+
 #include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
 #include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
 #include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
@@ -331,6 +334,23 @@ Future<Nothing> StoreProcess::moveLayer(
       flags.docker_store_dir,
       layerId);
 
+  const string sourceRootfs = paths::getImageLayerRootfsPath(
+      source,
+      flags.image_provisioner_backend);
+
+#ifdef __linux__
+  // If the backend is "overlay", we need to convert
+  // AUFS whiteout files to OverlayFS whiteout files.
+  if (flags.image_provisioner_backend == OVERLAY_BACKEND) {
+    Try<Nothing> convert = convertWhiteouts(sourceRootfs);
+    if (convert.isError()) {
+      return Failure(
+          "Failed to convert the whiteout files under '" +
+          sourceRootfs + "': " + convert.error());
+    }
+  }
+#endif
+
   if (!os::exists(target)) {
     // This is the case that we pull the layer for the first time.
     Try<Nothing> mkdir = os::mkdir(target);
@@ -349,10 +369,6 @@ Future<Nothing> StoreProcess::moveLayer(
   } else {
     // This is the case where the layer has already been pulled with a
     // different backend.
-    const string sourceRootfs = paths::getImageLayerRootfsPath(
-        source,
-        flags.image_provisioner_backend);
-
     Try<Nothing> rename = os::rename(sourceRootfs, targetRootfs);
     if (rename.isError()) {
       return Failure(

http://git-wip-us.apache.org/repos/asf/mesos/blob/3800cfd0/src/slave/containerizer/mesos/provisioner/utils.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/utils.cpp b/src/slave/containerizer/mesos/provisioner/utils.cpp
new file mode 100644
index 0000000..340cf48
--- /dev/null
+++ b/src/slave/containerizer/mesos/provisioner/utils.cpp
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "slave/containerizer/mesos/provisioner/utils.hpp"
+
+using std::string;
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+#ifdef __linux__
+// This function is used to convert AUFS whiteouts to OverlayFS whiteouts and
+// the AUFS whiteouts will be removed after the conversion is done. Whiteout is
+// for hiding directories and files in the lower level layers. AUFS whiteout is
+// the whiteout standard in Docker, and it relies on '.wh..wh..opq' to hide
+// directory and '.wh.<filename>' to hide file, whereas OverlayFS relies on
+// setting an extended attribute 'trusted.overlay.opaque' on the directory to
+// hide it and creating character device with 0/0 device number to hide file.
+// See the links below for more details:
+// http://aufs.sourceforge.net/aufs.html
+// https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt
+Try<Nothing> convertWhiteouts(const string& directory)
+{
+  char* rootfs[] = {const_cast<char*>(directory.c_str()), nullptr};
+
+  FTS* tree = ::fts_open(rootfs, FTS_NOCHDIR | FTS_PHYSICAL, nullptr);
+  if (tree == nullptr) {
+    return Error("Failed to open '" + directory + "': " + os::strerror(errno));
+  }
+
+  for (FTSENT *node = ::fts_read(tree);
+       node != nullptr; node = ::fts_read(tree)) {
+    if (node->fts_info != FTS_F) {
+      continue;
+    }
+
+    if (!strings::startsWith(node->fts_name, docker::spec::WHITEOUT_PREFIX)) {
+      continue;
+    }
+
+    const Path path = Path(node->fts_path);
+    if (node->fts_name == string(docker::spec::WHITEOUT_OPAQUE_PREFIX)) {
+      Try<Nothing> setxattr = os::setxattr(
+          path.dirname(),
+          "trusted.overlay.opaque",
+          "y",
+          0);
+
+      if (setxattr.isError()) {
+        ::fts_close(tree);
+        return Error(
+            "Failed to set extended attribute 'trusted.overlay.opaque'"
+            " for the directory '" + path.dirname() + "': " +
+            setxattr.error());
+      }
+    } else {
+      const string originalPath = path::join(
+          path.dirname(),
+          path.basename().substr(strlen(docker::spec::WHITEOUT_PREFIX)));
+
+      Try<Nothing> mknod = os::mknod(originalPath, S_IFCHR, 0);
+      if (mknod.isError()) {
+        ::fts_close(tree);
+        return Error(
+            "Failed to create character device '" +
+            originalPath + "': " + mknod.error());
+      }
+    }
+
+    // Remove the AUFS whiteout file.
+    Try<Nothing> rm = os::rm(path.string());
+    if (rm.isError()) {
+      ::fts_close(tree);
+      return Error(
+          "Failed to remove AUFS whiteout file '" +
+          path.string() + "': " + rm.error());
+    }
+  }
+
+  if (errno != 0) {
+    Error error = ErrnoError();
+    ::fts_close(tree);
+    return error;
+  }
+
+  if (::fts_close(tree) != 0) {
+    return Error(
+        "Failed to stop traversing file system: " + os::strerror(errno));
+  }
+
+  return Nothing();
+}
+#endif
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {

http://git-wip-us.apache.org/repos/asf/mesos/blob/3800cfd0/src/slave/containerizer/mesos/provisioner/utils.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/utils.hpp b/src/slave/containerizer/mesos/provisioner/utils.hpp
new file mode 100644
index 0000000..4efce0e
--- /dev/null
+++ b/src/slave/containerizer/mesos/provisioner/utils.hpp
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __MESOS_CONTAINERIZER_UTILS_HPP__
+#define __MESOS_CONTAINERIZER_UTILS_HPP__
+
+#include <stout/os.hpp>
+
+#include <mesos/docker/spec.hpp>
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+#ifdef __linux__
+Try<Nothing> convertWhiteouts(const std::string& directory);
+#endif
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __MESOS_CONTAINERIZER_UTILS_HPP__


Mime
View raw message