src/tests/containerizer/isolator_tests.cpp

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <unistd.h>

#include <string>

#include <process/future.hpp>
#include <process/owned.hpp>

#include <stout/gtest.hpp>
#include <stout/os.hpp>
#include <stout/path.hpp>
#include <stout/stringify.hpp>
#include <stout/strings.hpp>
#include <stout/uuid.hpp>

#include <mesos/mesos.hpp>

#include <mesos/slave/containerizer.hpp>

#ifdef __linux__
#include "linux/ns.hpp"
#endif

#include "slave/containerizer/fetcher.hpp"

#include "slave/containerizer/mesos/containerizer.hpp"
#include "slave/containerizer/mesos/paths.hpp"

#include "tests/mesos.hpp"

using std::string;

using process::Future;
using process::Owned;

using mesos::internal::slave::Containerizer;
using mesos::internal::slave::Fetcher;
using mesos::internal::slave::MesosContainerizer;

using mesos::internal::slave::containerizer::paths::getSandboxPath;

using mesos::slave::ContainerClass;
using mesos::slave::ContainerTermination;

namespace mesos {
namespace internal {
namespace tests {

#ifdef __linux__
class NamespacesIsolatorTest : public MesosTest
{
public:
  void SetUp() override
  {
    MesosTest::SetUp();

    directory = os::getcwd(); // We're inside a temporary sandbox.
    containerId.set_value(id::UUID::random().toString());
  }

  Try<Owned<MesosContainerizer>> createContainerizer(
      const string& isolation,
      const Option<bool>& disallowSharingAgentPidNamespace = None(),
      const Option<bool>& disallowSharingAgentIpcNamespace = None(),
      const Option<Bytes>& defaultContainerShmSize = None())
  {
    slave::Flags flags = CreateSlaveFlags();
    flags.image_providers = "docker";
    flags.isolation = isolation + ",docker/runtime";

    if (disallowSharingAgentPidNamespace.isSome()) {
      flags.disallow_sharing_agent_pid_namespace =
        disallowSharingAgentPidNamespace.get();
    }

    if (disallowSharingAgentIpcNamespace.isSome()) {
      flags.disallow_sharing_agent_ipc_namespace =
        disallowSharingAgentIpcNamespace.get();
    }

    flags.default_container_shm_size = defaultContainerShmSize;

    fetcher.reset(new Fetcher(flags));

    Try<MesosContainerizer*> _containerizer =
      MesosContainerizer::create(flags, true, fetcher.get());

    if (_containerizer.isError()) {
      return Error(_containerizer.error());
    }

    return Owned<MesosContainerizer>(_containerizer.get());
  }

  // Read a uint64_t value from the given path.
  Try<uint64_t> readValue(const string& path)
  {
    Try<string> value = os::read(path);

    if (value.isError()) {
      return Error("Failed to read '" + path + "': " + value.error());
    }

    return numify<uint64_t>(strings::trim(value.get()));
  }

  string directory;
  Owned<Fetcher> fetcher;
  ContainerID containerId;
};


TEST_F(NamespacesIsolatorTest, ROOT_PidNamespace)
{
  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer("filesystem/linux,namespaces/pid");

  ASSERT_SOME(containerizer);

  // Write the command's pid namespace inode and init name to files.
  const string command =
    "stat -Lc %i /proc/1/ns/pid > ns && (cat /proc/1/cmdline > init)";

  process::Future<Containerizer::LaunchResult> launch =
    containerizer.get()->launch(
        containerId,
        createContainerConfig(
            None(),
            createExecutorInfo("executor", command),
            directory),
        std::map<string, string>(),
        None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  // Wait on the container.
  Future<Option<ContainerTermination>> wait =
    containerizer.get()->wait(containerId);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());

  // Check the executor exited correctly.
  EXPECT_TRUE(wait->get().has_status());
  EXPECT_EQ(0, wait->get().status());

  // Check that the command was run in a different pid namespace.
  Result<ino_t> testPidNamespace = ns::getns(::getpid(), "pid");
  ASSERT_SOME(testPidNamespace);

  Try<string> containerPidNamespace = os::read(path::join(directory, "ns"));
  ASSERT_SOME(containerPidNamespace);

  EXPECT_NE(stringify(testPidNamespace.get()),
            strings::trim(containerPidNamespace.get()));

  // Check that the word 'mesos' is the part of the name for the
  // container's 'init' process. This verifies that /proc has been
  // correctly mounted for the container.
  Try<string> init = os::read(path::join(directory, "init"));
  ASSERT_SOME(init);

  EXPECT_TRUE(strings::contains(init.get(), "mesos"));
}


// This test verifies a top-level container can share pid namespace
// with the agent when the field `share_pid_namespace` is set as
// true in `ContainerInfo.linux_info`. Please note that the agent flag
// `--disallow_sharing_agent_pid_namespace` is set to
// false by default, that means top-level container is allowed to share
// pid namespace with agent.
TEST_F(NamespacesIsolatorTest, ROOT_SharePidNamespace)
{
  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer("filesystem/linux,namespaces/pid");

  ASSERT_SOME(containerizer);

  // Write the command's pid namespace inode to file.
  const string command = "stat -Lc %i /proc/1/ns/pid > ns";

  mesos::slave::ContainerConfig containerConfig = createContainerConfig(
      None(),
      createExecutorInfo("executor", command),
      directory);

  ContainerInfo* container = containerConfig.mutable_container_info();
  container->set_type(ContainerInfo::MESOS);
  container->mutable_linux_info()->set_share_pid_namespace(true);

  process::Future<Containerizer::LaunchResult> launch =
    containerizer.get()->launch(
        containerId,
        containerConfig,
        std::map<string, string>(),
        None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  // Wait on the container.
  Future<Option<ContainerTermination>> wait =
    containerizer.get()->wait(containerId);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());

  // Check the executor exited correctly.
  EXPECT_TRUE(wait->get().has_status());
  EXPECT_EQ(0, wait->get().status());

  // Check that the command was run in the same pid namespace.
  Result<ino_t> testPidNamespace = ns::getns(::getpid(), "pid");
  ASSERT_SOME(testPidNamespace);

  Try<string> containerPidNamespace = os::read(path::join(directory, "ns"));
  ASSERT_SOME(containerPidNamespace);

  EXPECT_EQ(stringify(testPidNamespace.get()),
            strings::trim(containerPidNamespace.get()));
}


// This test verifies launching a top-level container to share
// pid namespace with agent will fail when the agent flag
// `--disallow_sharing_agent_pid_namespace` is set to true.
TEST_F(NamespacesIsolatorTest, ROOT_SharePidNamespaceWhenDisallow)
{
  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer("filesystem/linux,namespaces/pid", true);

  ASSERT_SOME(containerizer);

  const string command = "sleep 1000";

  mesos::slave::ContainerConfig containerConfig = createContainerConfig(
      None(),
      createExecutorInfo("executor", command),
      directory);

  ContainerInfo* container = containerConfig.mutable_container_info();
  container->set_type(ContainerInfo::MESOS);
  container->mutable_linux_info()->set_share_pid_namespace(true);

  process::Future<Containerizer::LaunchResult> launch =
    containerizer.get()->launch(
        containerId,
        containerConfig,
        std::map<string, string>(),
        None());

  AWAIT_FAILED(launch);
}


// This test verifies that when `namespaces/ipc` isolator is enabled and
// container's IPC mode is not set, for backward compatibility we will
// keep the previous behavior: Top level container will have its own IPC
// namespace and nested container will share the IPC namespace with its
// parent container. If the container does not have its own rootfs, it
// will share agent's /dev/shm, otherwise it will have its own /dev/shm.
TEST_F(NamespacesIsolatorTest, ROOT_INTERNET_IPCNamespaceWithIPCModeUnset)
{
  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer("filesystem/linux,namespaces/ipc");

  ASSERT_SOME(containerizer);

  // Value we will set the top-level container's IPC namespace shmmax to.
  uint64_t shmmaxValue = static_cast<uint64_t>(::getpid());

  Try<uint64_t> hostShmmax = readValue("/proc/sys/kernel/shmmax");
  ASSERT_SOME(hostShmmax);

  // Verify that the host namespace shmmax is different.
  ASSERT_NE(hostShmmax.get(), shmmaxValue);

  // Launch a top-level container with IPC mode
  // unset and it does not have its own rootfs.
  string command =
    "stat -Lc %i /proc/self/ns/ipc > ns && "
    "echo " + stringify(shmmaxValue) + " > /proc/sys/kernel/shmmax && "
    "cp /proc/sys/kernel/shmmax /dev/shm/shmmax && "
    "sleep 1000";

  process::Future<Containerizer::LaunchResult> launch =
    containerizer.get()->launch(
        containerId,
        createContainerConfig(
            None(),
            createExecutorInfo("executor", command),
            directory),
        std::map<string, string>(),
        None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  // Since the top-level container does not have its own
  // rootfs, it will share host's /dev/shm, so let's wait
  // until /dev/shm/shmmax is created in the host.
  Duration waited = Duration::zero();

  do {
    if (os::exists("/dev/shm/shmmax")) {
      break;
    }

    os::sleep(Seconds(1));
    waited += Seconds(1);
  } while (waited < process::TEST_AWAIT_TIMEOUT);

  EXPECT_LT(waited, process::TEST_AWAIT_TIMEOUT);

  // Launch a nested container with IPC mode unset and it has its own rootfs.
  ContainerID nestedContainerId;
  nestedContainerId.mutable_parent()->CopyFrom(containerId);
  nestedContainerId.set_value(id::UUID::random().toString());

  mesos::Image image;
  image.set_type(mesos::Image::DOCKER);
  image.mutable_docker()->set_name("alpine");

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.mutable_mesos()->mutable_image()->CopyFrom(image);

  command =
    "stat -Lc %i /proc/self/ns/ipc > ns && "
    "test `cat /proc/sys/kernel/shmmax` = " + stringify(shmmaxValue) + " && "
    "touch /dev/shm/file";

  launch = containerizer.get()->launch(
      nestedContainerId,
      createContainerConfig(createCommandInfo(command), containerInfo),
      std::map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  // Wait on the nested container.
  Future<Option<ContainerTermination>> wait =
    containerizer.get()->wait(nestedContainerId);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  EXPECT_TRUE(wait->get().has_status());
  EXPECT_EQ(0, wait->get().status());

  // Check that top-level container and the nested container are in the
  // same IPC namespace but not in the same IPC namespace with host.
  Result<ino_t> testIPCNamespace = ns::getns(::getpid(), "ipc");
  ASSERT_SOME(testIPCNamespace);

  Try<string> containerIPCNamespace = os::read(path::join(directory, "ns"));
  ASSERT_SOME(containerIPCNamespace);

  Try<string> nestedcontainerIPCNamespace =
    os::read(path::join(getSandboxPath(directory, nestedContainerId), "ns"));

  ASSERT_SOME(nestedcontainerIPCNamespace);

  EXPECT_NE(stringify(testIPCNamespace.get()),
            strings::trim(containerIPCNamespace.get()));

  EXPECT_EQ(strings::trim(containerIPCNamespace.get()),
            strings::trim(nestedcontainerIPCNamespace.get()));

  // The nested container will have its own /dev/shm since it has its own
  // rootfs, so the file it created should not exist in the host.
  ASSERT_FALSE(os::exists("/dev/shm/file"));

  // Check that we modified the IPC shmmax of the namespace, not the host.
  Try<uint64_t> containerShmmax = readValue("/dev/shm/shmmax");
  ASSERT_SOME(containerShmmax);

  // Verify that we didn't modify shmmax in the host namespace.
  ASSERT_EQ(hostShmmax.get(), readValue("/proc/sys/kernel/shmmax").get());

  EXPECT_NE(hostShmmax.get(), containerShmmax.get());
  EXPECT_EQ(shmmaxValue, containerShmmax.get());

  Future<Option<ContainerTermination>> termination =
    containerizer.get()->destroy(containerId);

  AWAIT_READY(termination);
  ASSERT_SOME(termination.get());
  ASSERT_TRUE(termination.get()->has_status());
  EXPECT_WTERMSIG_EQ(SIGKILL, termination.get()->status());

  ASSERT_SOME(os::rm("/dev/shm/shmmax"));
}


// This test verifies that when `namespaces/ipc` isolator is not enabled,
// for backward compatibility we will keep the previous behavior: Any
// containers will share IPC namespace with agent, and if the container
// does not have its own rootfs, it will also share agent's /dev/shm,
// otherwise it will have its own /dev/shm.
TEST_F(NamespacesIsolatorTest,
       ROOT_INTERNET_IPCNamespaceWithIPCIsolatorDisabled)
{
  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer("filesystem/linux");

  ASSERT_SOME(containerizer);

  // Launch a top-level container which does not have its own rootfs.
  string command =
    "stat -Lc %i /proc/self/ns/ipc > ns && "
    "touch /dev/shm/root && "
    "sleep 1000";

  process::Future<Containerizer::LaunchResult> launch =
    containerizer.get()->launch(
        containerId,
        createContainerConfig(
            None(),
            createExecutorInfo("executor", command),
            directory),
        std::map<string, string>(),
        None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  // Since the top-level container does not have its own
  // rootfs, it will share host's /dev/shm, so let's wait
  // until /dev/shm/root is created in the host.
  Duration waited = Duration::zero();

  do {
    if (os::exists("/dev/shm/root")) {
      break;
    }

    os::sleep(Seconds(1));
    waited += Seconds(1);
  } while (waited < process::TEST_AWAIT_TIMEOUT);

  EXPECT_LT(waited, process::TEST_AWAIT_TIMEOUT);

  // Launch a nested container which has its own rootfs.
  ContainerID nestedContainerId;
  nestedContainerId.mutable_parent()->CopyFrom(containerId);
  nestedContainerId.set_value(id::UUID::random().toString());

  mesos::Image image;
  image.set_type(mesos::Image::DOCKER);
  image.mutable_docker()->set_name("alpine");

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.mutable_mesos()->mutable_image()->CopyFrom(image);

  command =
    "stat -Lc %i /proc/self/ns/ipc > ns && "
    "touch /dev/shm/nested";

  launch = containerizer.get()->launch(
      nestedContainerId,
      createContainerConfig(createCommandInfo(command), containerInfo),
      std::map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  // Wait on the nested container.
  Future<Option<ContainerTermination>> wait =
    containerizer.get()->wait(nestedContainerId);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  EXPECT_TRUE(wait->get().has_status());
  EXPECT_EQ(0, wait->get().status());

  // Check that top-level container and the nested container are
  // in the same IPC namespace with host.
  Result<ino_t> testIPCNamespace = ns::getns(::getpid(), "ipc");
  ASSERT_SOME(testIPCNamespace);

  Try<string> containerIPCNamespace = os::read(path::join(directory, "ns"));
  ASSERT_SOME(containerIPCNamespace);

  Try<string> nestedcontainerIPCNamespace =
    os::read(path::join(getSandboxPath(directory, nestedContainerId), "ns"));

  ASSERT_SOME(nestedcontainerIPCNamespace);

  EXPECT_EQ(stringify(testIPCNamespace.get()),
            strings::trim(containerIPCNamespace.get()));

  EXPECT_EQ(strings::trim(containerIPCNamespace.get()),
            strings::trim(nestedcontainerIPCNamespace.get()));

  // The nested container will have its own /dev/shm since it has its own
  // rootfs, so the file it created should not exist in the host.
  ASSERT_FALSE(os::exists("/dev/shm/nested"));

  Future<Option<ContainerTermination>> termination =
    containerizer.get()->destroy(containerId);

  AWAIT_READY(termination);
  ASSERT_SOME(termination.get());
  ASSERT_TRUE(termination.get()->has_status());
  EXPECT_WTERMSIG_EQ(SIGKILL, termination.get()->status());

  ASSERT_SOME(os::rm("/dev/shm/root"));
}


// This test verifies that a top-level container with private IPC mode will
// have its own IPC namespace and /dev/shm, and it can share IPC namespace
// and /dev/shm with its child containers, grandchild containers and debug
// containers.
TEST_F(NamespacesIsolatorTest, ROOT_INTERNET_ShareIPCNamespace)
{
  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer("filesystem/linux,namespaces/ipc");

  ASSERT_SOME(containerizer);

  // Launch a top-level container with `PRIVATE` IPC mode and 128MB /dev/shm,
  // check its /dev/shm size is correctly set and its IPC namespace is
  // different than agent's IPC namespace, write its IPC namespace inode to
  // a file under /dev/shm.
  const string command =
    "df -m /dev/shm | grep -w 128 && "
    "test `stat -Lc %i /proc/self/ns/ipc` != `stat -Lc %i /proc/1/ns/ipc` && "
    "stat -Lc %i /proc/self/ns/ipc > /dev/shm/root && "
    "touch marker && "
    "sleep 1000";

  mesos::slave::ContainerConfig containerConfig = createContainerConfig(
      None(),
      createExecutorInfo("executor", command),
      directory);

  ContainerInfo* container = containerConfig.mutable_container_info();
  container->set_type(ContainerInfo::MESOS);
  container->mutable_linux_info()->set_ipc_mode(LinuxInfo::PRIVATE);
  container->mutable_linux_info()->set_shm_size(128);

  process::Future<Containerizer::LaunchResult> launch =
    containerizer.get()->launch(
        containerId,
        containerConfig,
        std::map<string, string>(),
        None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  // Wait until the marker file is created.
  Duration waited = Duration::zero();

  do {
    if (os::exists(path::join(directory, "marker"))) {
      break;
    }

    os::sleep(Seconds(1));
    waited += Seconds(1);
  } while (waited < process::TEST_AWAIT_TIMEOUT);

  EXPECT_LT(waited, process::TEST_AWAIT_TIMEOUT);

  // The file created by the top-level container should only exist in
  // its own /dev/shm rather than in agent's /dev/shm.
  ASSERT_FALSE(os::exists("/dev/shm/root"));

  // Now launch two child containers with `SHARE_PARENT` ipc mode.
  ContainerID childContainerId1, childContainerId2;

  childContainerId1.mutable_parent()->CopyFrom(containerId);
  childContainerId1.set_value(id::UUID::random().toString());

  childContainerId2.mutable_parent()->CopyFrom(containerId);
  childContainerId2.set_value(id::UUID::random().toString());

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.mutable_linux_info()->set_ipc_mode(LinuxInfo::SHARE_PARENT);

  // Launch the first child container, check its /dev/shm size is 128MB, it
  // can see the file created by its parent container in /dev/shm and it is
  // in the same IPC namespace with its parent container, and then write its
  // IPC namespace inode to a file under /dev/shm.
  launch = containerizer.get()->launch(
      childContainerId1,
      createContainerConfig(
          createCommandInfo(
              "df -m /dev/shm | grep -w 128 && "
              "test `stat -Lc %i /proc/self/ns/ipc` = `cat /dev/shm/root` && "
              "stat -Lc %i /proc/self/ns/ipc > /dev/shm/child1 && "
              "touch marker && "
              "sleep 1000"),
          containerInfo),
      std::map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  // Wait until the marker file is created.
  waited = Duration::zero();
  const string childSandboxPath1 = getSandboxPath(directory, childContainerId1);

  do {
    if (os::exists(path::join(childSandboxPath1, "marker"))) {
      break;
    }

    os::sleep(Seconds(1));
    waited += Seconds(1);
  } while (waited < process::TEST_AWAIT_TIMEOUT);

  EXPECT_LT(waited, process::TEST_AWAIT_TIMEOUT);

  // Launch the second child container with its own rootfs, check its /dev/shm
  // size is 128MB, it can see the files created by its parent container and the
  // first child container in /dev/shm and it is in the same IPC namespace with
  // its parent container and the first child container, and then write its IPC
  // namespace inode to a file under /dev/shm.
  mesos::Image image;
  image.set_type(mesos::Image::DOCKER);
  image.mutable_docker()->set_name("alpine");

  containerInfo.mutable_mesos()->mutable_image()->CopyFrom(image);

  launch = containerizer.get()->launch(
      childContainerId2,
      createContainerConfig(
          createCommandInfo(
              "df -m /dev/shm | grep -w 128 && "
              "test `stat -Lc %i /proc/self/ns/ipc` = `cat /dev/shm/root` && "
              "test `stat -Lc %i /proc/self/ns/ipc` = `cat /dev/shm/child1` && "
              "stat -Lc %i /proc/self/ns/ipc > /dev/shm/child2 && "
              "touch marker && "
              "sleep 1000"),
          containerInfo),
      std::map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  // Wait until the marker file is created.
  waited = Duration::zero();
  const string childSandboxPath2 = getSandboxPath(directory, childContainerId2);

  do {
    if (os::exists(path::join(childSandboxPath2, "marker"))) {
      break;
    }

    os::sleep(Seconds(1));
    waited += Seconds(1);
  } while (waited < process::TEST_AWAIT_TIMEOUT);

  EXPECT_LT(waited, process::TEST_AWAIT_TIMEOUT);

  // Launch a grandchild container with `SHARE_PARENT` ipc mode under the first
  // child container, check its /dev/shm size is 128MB, it can see the files
  // created by its parent and grandparent containers and it is in the same IPC
  // namespace with its parent and grandparent containers.
  ContainerID grandchildContainerId;
  grandchildContainerId.mutable_parent()->CopyFrom(childContainerId1);
  grandchildContainerId.set_value(id::UUID::random().toString());

  launch = containerizer.get()->launch(
      grandchildContainerId,
      createContainerConfig(
          createCommandInfo(
              "df -m /dev/shm | grep -w 128 && "
              "test `stat -Lc %i /proc/self/ns/ipc` = `cat /dev/shm/child1` && "
              "test `stat -Lc %i /proc/self/ns/ipc` = `cat /dev/shm/root`"),
          containerInfo),
      std::map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  Future<Option<ContainerTermination>> wait =
    containerizer.get()->wait(grandchildContainerId);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  ASSERT_TRUE(wait.get()->has_status());
  EXPECT_WEXITSTATUS_EQ(0, wait.get()->status());

  // Launch a debug container under the first child container, check its
  // /dev/shm size is 128MB and it is in the same IPC namespace with its
  // parent container.
  ContainerID debugContainerId1;
  debugContainerId1.mutable_parent()->CopyFrom(childContainerId1);
  debugContainerId1.set_value(id::UUID::random().toString());

  containerInfo.clear_mesos();
  containerInfo.clear_linux_info();

  launch = containerizer.get()->launch(
      debugContainerId1,
      createContainerConfig(
          createCommandInfo(
              "df -m /dev/shm | grep -w 128 && "
              "test `stat -Lc %i /proc/self/ns/ipc` = `cat /dev/shm/child1`"),
          containerInfo,
          ContainerClass::DEBUG),
      std::map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  wait = containerizer.get()->wait(debugContainerId1);
  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  ASSERT_TRUE(wait.get()->has_status());
  EXPECT_WEXITSTATUS_EQ(0, wait.get()->status());

  // Launch another debug container under the second child container, check its
  // /dev/shm size is 128MB and it is in the same IPC namespace with its parent
  // container.
  ContainerID debugContainerId2;
  debugContainerId2.mutable_parent()->CopyFrom(childContainerId2);
  debugContainerId2.set_value(id::UUID::random().toString());

  launch = containerizer.get()->launch(
      debugContainerId2,
      createContainerConfig(
          createCommandInfo(
              "df -m /dev/shm | grep -w 128 && "
              "test `stat -Lc %i /proc/self/ns/ipc` = `cat /dev/shm/child2`"),
          containerInfo,
          ContainerClass::DEBUG),
      std::map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  wait = containerizer.get()->wait(debugContainerId2);
  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  ASSERT_TRUE(wait.get()->has_status());
  EXPECT_WEXITSTATUS_EQ(0, wait.get()->status());

  Future<Option<ContainerTermination>> termination =
    containerizer.get()->destroy(containerId);

  AWAIT_READY(termination);
  ASSERT_SOME(termination.get());
  ASSERT_TRUE(termination.get()->has_status());
  EXPECT_WTERMSIG_EQ(SIGKILL, termination.get()->status());
}


// This test verifies that nested container with private IPC mode will
// have its own IPC namespace and /dev/shm.
TEST_F(NamespacesIsolatorTest, ROOT_PrivateIPCNamespace)
{
  // Create containerizer with `--default_container_shm_size=64MB`.
  Try<Owned<MesosContainerizer>> containerizer = createContainerizer(
      "filesystem/linux,namespaces/ipc",
      None(),
      None(),
      Megabytes(64));

  ASSERT_SOME(containerizer);

  // Launch a top-level container with `PRIVATE` IPC mode, check its /dev/shm
  // size is correctly set to the default value, touch a file in its /dev/shm,
  // and write its IPC namespace inode to a file in its sandbox.
  const string command =
    "df -m /dev/shm | grep -w 64 && "
    "touch /dev/shm/root &&"
    "stat -Lc %i /proc/self/ns/ipc > ns && "
    "sleep 1000";

  mesos::slave::ContainerConfig containerConfig = createContainerConfig(
      None(),
      createExecutorInfo("executor", command),
      directory);

  ContainerInfo* container = containerConfig.mutable_container_info();
  container->set_type(ContainerInfo::MESOS);
  container->mutable_linux_info()->set_ipc_mode(LinuxInfo::PRIVATE);

  process::Future<Containerizer::LaunchResult> launch =
    containerizer.get()->launch(
        containerId,
        containerConfig,
        std::map<string, string>(),
        None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  // Wait until the `ns` file is created in the sandbox.
  Duration waited = Duration::zero();

  do {
    if (os::exists(path::join(directory, "ns"))) {
      break;
    }

    os::sleep(Seconds(1));
    waited += Seconds(1);
  } while (waited < process::TEST_AWAIT_TIMEOUT);

  EXPECT_LT(waited, process::TEST_AWAIT_TIMEOUT);

  // Launch a nested container with `PRIVATE` IPC mode, check its /dev/shm
  // size is correctly set to the default value and the file created by the
  // top-level container does not exist in its /dev/shm, touch a file in its
  // /dev/shm and write its IPC namespace inode to a file in its sandbox.
  ContainerID nestedContainerId1;
  nestedContainerId1.mutable_parent()->CopyFrom(containerId);
  nestedContainerId1.set_value(id::UUID::random().toString());

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.mutable_linux_info()->set_ipc_mode(LinuxInfo::PRIVATE);

  launch = containerizer.get()->launch(
      nestedContainerId1,
      createContainerConfig(
          createCommandInfo(
              "df -m /dev/shm | grep -w 64 &&"
              "test ! -e /dev/shm/root &&"
              "touch /dev/shm/nested1 &&"
              "stat -Lc %i /proc/self/ns/ipc > ns && "
              "sleep 1000"),
          containerInfo),
      std::map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  // Wait until the `ns` file is created in the sandbox.
  waited = Duration::zero();
  const string nestedSandboxPath1 =
    getSandboxPath(directory, nestedContainerId1);

  do {
    if (os::exists(path::join(nestedSandboxPath1, "ns"))) {
      break;
    }

    os::sleep(Seconds(1));
    waited += Seconds(1);
  } while (waited < process::TEST_AWAIT_TIMEOUT);

  EXPECT_LT(waited, process::TEST_AWAIT_TIMEOUT);

  // Launch another nested container with private IPC mode and 128MB
  // /dev/shm, check its /dev/shm size is correctly set to 128MB and
  // the files created by the top-level container and the first nested
  // container do not exist in its /dev/shm, write its IPC namespace
  // inode to a file in its sandbox.
  ContainerID nestedContainerId2;
  nestedContainerId2.mutable_parent()->CopyFrom(containerId);
  nestedContainerId2.set_value(id::UUID::random().toString());

  containerInfo.mutable_linux_info()->set_shm_size(128);

  launch = containerizer.get()->launch(
      nestedContainerId2,
      createContainerConfig(
          createCommandInfo(
              "df -m /dev/shm | grep -w 128 &&"
              "test ! -e /dev/shm/root &&"
              "test ! -e /dev/shm/nested1 &&"
              "stat -Lc %i /proc/self/ns/ipc > ns"),
          containerInfo),
      std::map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  Future<Option<ContainerTermination>> wait = containerizer.get()->wait(
      nestedContainerId2);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  ASSERT_TRUE(wait.get()->has_status());
  EXPECT_WEXITSTATUS_EQ(0, wait.get()->status());

  // Check top-level container and the two nested containers
  // have their own IPC namespaces.
  Try<uint64_t> rootIpcNamespace = readValue(path::join(directory, "ns"));
  ASSERT_SOME(rootIpcNamespace);

  Try<uint64_t> nestedIpcNamespace1 =
    readValue(path::join(nestedSandboxPath1, "ns"));

  ASSERT_SOME(nestedIpcNamespace1);

  Try<uint64_t> nestedIpcNamespace2 =
    readValue(path::join(getSandboxPath(directory, nestedContainerId2), "ns"));

  ASSERT_SOME(nestedIpcNamespace2);

  EXPECT_NE(rootIpcNamespace.get(), nestedIpcNamespace1.get());
  EXPECT_NE(rootIpcNamespace.get(), nestedIpcNamespace2.get());
  EXPECT_NE(nestedIpcNamespace1.get(), nestedIpcNamespace2.get());

  Future<Option<ContainerTermination>> termination =
    containerizer.get()->destroy(containerId);

  AWAIT_READY(termination);
  ASSERT_SOME(termination.get());
  ASSERT_TRUE(termination.get()->has_status());
  EXPECT_WTERMSIG_EQ(SIGKILL, termination.get()->status());
}


// This test verifies that top-level container and nested
// containers can share agent's IPC namespace and /dev/shm.
TEST_F(NamespacesIsolatorTest, ROOT_INTERNET_ShareAgentIPCNamespace)
{
  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer("filesystem/linux,namespaces/ipc");

  ASSERT_SOME(containerizer);

  // Launch a top-level container with `SHARE_PARENT` IPC mode,
  // write its IPC namespace inode to a file under /dev/shm.
  const string command =
    "stat -Lc %i /proc/self/ns/ipc > /dev/shm/root && "
    "sleep 1000";

  mesos::slave::ContainerConfig containerConfig = createContainerConfig(
      None(),
      createExecutorInfo("executor", command),
      directory);

  ContainerInfo* container = containerConfig.mutable_container_info();
  container->set_type(ContainerInfo::MESOS);
  container->mutable_linux_info()->set_ipc_mode(LinuxInfo::SHARE_PARENT);

  process::Future<Containerizer::LaunchResult> launch =
    containerizer.get()->launch(
        containerId,
        containerConfig,
        std::map<string, string>(),
        None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  // Launch a nested container with `SHARE_PARENT` IPC mode,
  // write its IPC namespace inode to a file under /dev/shm.
  ContainerID nestedContainerId1;
  nestedContainerId1.mutable_parent()->CopyFrom(containerId);
  nestedContainerId1.set_value(id::UUID::random().toString());

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.mutable_linux_info()->set_ipc_mode(LinuxInfo::SHARE_PARENT);

  launch = containerizer.get()->launch(
      nestedContainerId1,
      createContainerConfig(
          createCommandInfo("stat -Lc %i /proc/self/ns/ipc > /dev/shm/nest1"),
          containerInfo),
      std::map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  Future<Option<ContainerTermination>> wait = containerizer.get()->wait(
      nestedContainerId1);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  ASSERT_TRUE(wait.get()->has_status());
  EXPECT_WEXITSTATUS_EQ(0, wait.get()->status());

  // Launch another nested container with `SHARE_PARENT` IPC mode and its
  // own rootfs, write its IPC namespace inode to a file under /dev/shm.
  ContainerID nestedContainerId2;
  nestedContainerId2.mutable_parent()->CopyFrom(containerId);
  nestedContainerId2.set_value(id::UUID::random().toString());

  mesos::Image image;
  image.set_type(mesos::Image::DOCKER);
  image.mutable_docker()->set_name("alpine");

  containerInfo.mutable_mesos()->mutable_image()->CopyFrom(image);

  launch = containerizer.get()->launch(
      nestedContainerId2,
      createContainerConfig(
          createCommandInfo("stat -Lc %i /proc/self/ns/ipc > /dev/shm/nest2"),
          containerInfo),
      std::map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  wait = containerizer.get()->wait(nestedContainerId2);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  ASSERT_TRUE(wait.get()->has_status());
  EXPECT_WEXITSTATUS_EQ(0, wait.get()->status());

  // Check top-level container and the two nested containers
  // share agent's IPC namespace and /dev/shm.
  Try<uint64_t> rootIpcNamespace = readValue("/dev/shm/root");
  ASSERT_SOME(rootIpcNamespace);

  Try<uint64_t> nestedIpcNamespace1 = readValue("/dev/shm/nest1");
  ASSERT_SOME(nestedIpcNamespace1);

  Try<uint64_t> nestedIpcNamespace2 = readValue("/dev/shm/nest2");
  ASSERT_SOME(nestedIpcNamespace2);

  Result<ino_t> agentIpcNamespace = ns::getns(::getpid(), "ipc");
  ASSERT_SOME(agentIpcNamespace);

  EXPECT_EQ(rootIpcNamespace.get(), agentIpcNamespace.get());
  EXPECT_EQ(nestedIpcNamespace1.get(), agentIpcNamespace.get());
  EXPECT_EQ(nestedIpcNamespace2.get(), agentIpcNamespace.get());

  Future<Option<ContainerTermination>> termination =
    containerizer.get()->destroy(containerId);

  AWAIT_READY(termination);
  ASSERT_SOME(termination.get());
  ASSERT_TRUE(termination.get()->has_status());
  EXPECT_WTERMSIG_EQ(SIGKILL, termination.get()->status());

  ASSERT_SOME(os::rm("/dev/shm/root"));
  ASSERT_SOME(os::rm("/dev/shm/nest1"));
  ASSERT_SOME(os::rm("/dev/shm/nest2"));
}


// This test verifies that top-level container with `SHARE_PARENT` IPC mode
// will fail to launch when `--disallow_sharing_agent_ipc_namespace = true`.
TEST_F(NamespacesIsolatorTest, ROOT_DisallowShareAgentIPCNamespace)
{
  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer("filesystem/linux,namespaces/ipc", None(), true);

  ASSERT_SOME(containerizer);

  // Launch a top-level container with `SHARE_PARENT` IPC mode.
  mesos::slave::ContainerConfig containerConfig = createContainerConfig(
      None(),
      createExecutorInfo("executor", "sleep 1000"),
      directory);

  ContainerInfo* container = containerConfig.mutable_container_info();
  container->set_type(ContainerInfo::MESOS);
  container->mutable_linux_info()->set_ipc_mode(LinuxInfo::SHARE_PARENT);

  process::Future<Containerizer::LaunchResult> launch =
    containerizer.get()->launch(
        containerId,
        containerConfig,
        std::map<string, string>(),
        None());

  AWAIT_FAILED(launch);
}


// This test verifies that we do not support specifying container's
// /dev/shm size when its IPC mode is not `PRIVATE`.
TEST_F(NamespacesIsolatorTest, ROOT_NonePrivateIPCModeWithShmSize)
{
  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer("filesystem/linux,namespaces/ipc");

  ASSERT_SOME(containerizer);

  // Launch a container with a specified /dev/shm size but without
  // specifying IPC mode.
  ContainerID containerId1;
  containerId1.set_value(id::UUID::random().toString());

  mesos::slave::ContainerConfig containerConfig = createContainerConfig(
      None(),
      createExecutorInfo("executor", "sleep 1000"),
      directory);

  ContainerInfo* container = containerConfig.mutable_container_info();
  container->set_type(ContainerInfo::MESOS);
  container->mutable_linux_info()->set_shm_size(128);

  process::Future<Containerizer::LaunchResult> launch =
    containerizer.get()->launch(
        containerId1,
        containerConfig,
        std::map<string, string>(),
        None());

  AWAIT_FAILED(launch);

  // Launch another container with a specified /dev/shm size and `SHARE_PARENT`
  // IPC mode.
  ContainerID containerId2;
  containerId2.set_value(id::UUID::random().toString());

  container->mutable_linux_info()->set_ipc_mode(LinuxInfo::SHARE_PARENT);

  launch = containerizer.get()->launch(
      containerId2,
      containerConfig,
      std::map<string, string>(),
      None());

  AWAIT_FAILED(launch);
}


// This test verifies that we do not support launching debug container
// with private IPC mode.
TEST_F(NamespacesIsolatorTest, ROOT_DebugContainerWithPrivateIPCMode)
{
  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer("filesystem/linux,namespaces/ipc");

  ASSERT_SOME(containerizer);

  // Launch a top-level container.
  process::Future<Containerizer::LaunchResult> launch =
    containerizer.get()->launch(
        containerId,
        createContainerConfig(
              None(),
              createExecutorInfo("executor", "sleep 1000"),
              directory),
        std::map<string, string>(),
        None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  // Launch a debug container with private IPC mode under the
  // top-level container.
  ContainerID debugContainerId;
  debugContainerId.mutable_parent()->CopyFrom(containerId);
  debugContainerId.set_value(id::UUID::random().toString());

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.mutable_linux_info()->set_ipc_mode(LinuxInfo::PRIVATE);

  launch = containerizer.get()->launch(
      debugContainerId,
      createContainerConfig(
          createCommandInfo("sleep 1000"),
          containerInfo,
          ContainerClass::DEBUG),
      std::map<string, string>(),
      None());

  AWAIT_FAILED(launch);

  Future<Option<ContainerTermination>> termination =
    containerizer.get()->destroy(containerId);

  AWAIT_READY(termination);
  ASSERT_SOME(termination.get());
  ASSERT_TRUE(termination.get()->has_status());
  EXPECT_WTERMSIG_EQ(SIGKILL, termination.get()->status());
}
#endif // __linux__

} // namespace tests {
} // namespace internal {
} // namespace mesos {