src/tests/containerizer/volume_host_path_isolator_tests.cpp

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <vector>

#include <mesos/mesos.hpp>

#include <process/owned.hpp>
#include <process/gtest.hpp>

#include <stout/format.hpp>
#include <stout/gtest.hpp>
#include <stout/os.hpp>
#include <stout/path.hpp>

#include "slave/containerizer/mesos/containerizer.hpp"

#include "slave/containerizer/mesos/isolators/volume/utils.hpp"

#include "tests/cluster.hpp"
#include "tests/mesos.hpp"

#include "tests/containerizer/docker_archive.hpp"

using process::Future;
using process::Owned;

using std::map;
using std::string;
using std::vector;

using testing::TestParamInfo;
using testing::Values;
using testing::WithParamInterface;

using mesos::internal::slave::Containerizer;
using mesos::internal::slave::Fetcher;
using mesos::internal::slave::MesosContainerizer;

using mesos::internal::slave::volume::HOST_PATH_WHITELIST_DELIM;

using mesos::master::detector::MasterDetector;

using mesos::slave::ContainerTermination;

namespace mesos {
namespace internal {
namespace tests {

class VolumeHostPathIsolatorTest : public MesosTest {};


// This test verifies that a volume with an absolute host path as
// well as an absolute container path is properly mounted in the
// container's mount namespace.
TEST_F(VolumeHostPathIsolatorTest, ROOT_VolumeFromHost)
{
  string registry = path::join(sandbox.get(), "registry");
  AWAIT_READY(DockerArchive::create(registry, "test_image"));

  slave::Flags flags = CreateSlaveFlags();
  flags.isolation = "filesystem/linux,docker/runtime";
  flags.docker_registry = registry;
  flags.docker_store_dir = path::join(sandbox.get(), "store");
  flags.image_providers = "docker";

  Fetcher fetcher(flags);

  Try<MesosContainerizer*> create =
    MesosContainerizer::create(flags, true, &fetcher);

  ASSERT_SOME(create);

  Owned<Containerizer> containerizer(create.get());

  ContainerID containerId;
  containerId.set_value(id::UUID::random().toString());

  ExecutorInfo executor = createExecutorInfo(
      "test_executor",
      "test -d /tmp/dir");

  executor.mutable_container()->CopyFrom(createContainerInfo(
      "test_image",
      {createVolumeHostPath("/tmp", sandbox.get(), Volume::RW)}));

  string dir = path::join(sandbox.get(), "dir");
  ASSERT_SOME(os::mkdir(dir));

  string directory = path::join(flags.work_dir, "sandbox");
  ASSERT_SOME(os::mkdir(directory));

  Future<Containerizer::LaunchResult> launch = containerizer->launch(
      containerId,
      createContainerConfig(None(), executor, directory),
      map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  Future<Option<ContainerTermination>> wait = containerizer->wait(containerId);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  ASSERT_TRUE(wait->get().has_status());
  EXPECT_WEXITSTATUS_EQ(0, wait->get().status());
}


// This test verifies that a container launched with a
// rootfs cannot write to a read-only HOST_PATH volume.
TEST_F(VolumeHostPathIsolatorTest, ROOT_ReadOnlyVolumeFromHost)
{
  string registry = path::join(sandbox.get(), "registry");
  AWAIT_READY(DockerArchive::create(registry, "test_image"));

  slave::Flags flags = CreateSlaveFlags();
  flags.isolation = "filesystem/linux,docker/runtime";
  flags.docker_registry = registry;
  flags.docker_store_dir = path::join(sandbox.get(), "store");
  flags.image_providers = "docker";

  Fetcher fetcher(flags);

  Try<MesosContainerizer*> create =
    MesosContainerizer::create(flags, true, &fetcher);

  ASSERT_SOME(create);

  Owned<Containerizer> containerizer(create.get());

  ContainerID containerId;
  containerId.set_value(id::UUID::random().toString());

  ExecutorInfo executor = createExecutorInfo(
      "test_executor",
      "echo abc > /tmp/dir/file");

  executor.mutable_container()->CopyFrom(createContainerInfo(
      "test_image",
      {createVolumeHostPath("/tmp", sandbox.get(), Volume::RO)}));

  string dir = path::join(sandbox.get(), "dir");
  ASSERT_SOME(os::mkdir(dir));

  string directory = path::join(flags.work_dir, "sandbox");
  ASSERT_SOME(os::mkdir(directory));

  Future<Containerizer::LaunchResult> launch = containerizer->launch(
      containerId,
      createContainerConfig(None(), executor, directory),
      map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  Future<Option<ContainerTermination>> wait = containerizer->wait(containerId);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  ASSERT_TRUE(wait->get().has_status());
  EXPECT_WEXITSTATUS_NE(0, wait->get().status());
}


// This test verifies that a file volume with an absolute host
// path as well as an absolute container path is properly mounted
// in the container's mount namespace.
TEST_F(VolumeHostPathIsolatorTest, ROOT_FileVolumeFromHost)
{
  string registry = path::join(sandbox.get(), "registry");
  AWAIT_READY(DockerArchive::create(registry, "test_image"));

  slave::Flags flags = CreateSlaveFlags();
  flags.isolation = "filesystem/linux,docker/runtime";
  flags.docker_registry = registry;
  flags.docker_store_dir = path::join(sandbox.get(), "store");
  flags.image_providers = "docker";

  Fetcher fetcher(flags);

  Try<MesosContainerizer*> create =
    MesosContainerizer::create(flags, true, &fetcher);

  ASSERT_SOME(create);

  Owned<Containerizer> containerizer(create.get());

  ContainerID containerId;
  containerId.set_value(id::UUID::random().toString());

  ExecutorInfo executor = createExecutorInfo(
      "test_executor",
      "test -f /tmp/test/file.txt");

  string file = path::join(sandbox.get(), "file");
  ASSERT_SOME(os::touch(file));

  executor.mutable_container()->CopyFrom(createContainerInfo(
      "test_image",
      {createVolumeHostPath("/tmp/test/file.txt", file, Volume::RW)}));

  string directory = path::join(flags.work_dir, "sandbox");
  ASSERT_SOME(os::mkdir(directory));

  Future<Containerizer::LaunchResult> launch = containerizer->launch(
      containerId,
      createContainerConfig(None(), executor, directory),
      map<string, string>(),
      None());

  AWAIT_ASSERT_EQ_FOR(
      Containerizer::LaunchResult::SUCCESS, launch, Seconds(60));

  Future<Option<ContainerTermination>> wait = containerizer->wait(containerId);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  ASSERT_TRUE(wait->get().has_status());
  EXPECT_WEXITSTATUS_EQ(0, wait->get().status());
}


// This test verifies that a volume with an absolute host path and a
// relative container path is properly mounted in the container's
// mount namespace. The mount point will be created in the sandbox.
TEST_F(VolumeHostPathIsolatorTest, ROOT_VolumeFromHostSandboxMountPoint)
{
  string registry = path::join(sandbox.get(), "registry");
  AWAIT_READY(DockerArchive::create(registry, "test_image"));

  slave::Flags flags = CreateSlaveFlags();
  flags.isolation = "filesystem/linux,docker/runtime";
  flags.docker_registry = registry;
  flags.docker_store_dir = path::join(sandbox.get(), "store");
  flags.image_providers = "docker";

  Fetcher fetcher(flags);

  Try<MesosContainerizer*> create =
    MesosContainerizer::create(flags, true, &fetcher);

  ASSERT_SOME(create);

  Owned<Containerizer> containerizer(create.get());

  ContainerID containerId;
  containerId.set_value(id::UUID::random().toString());

  ExecutorInfo executor = createExecutorInfo(
      "test_executor",
      "test -d mountpoint/dir");

  executor.mutable_container()->CopyFrom(createContainerInfo(
      "test_image",
      {createVolumeHostPath("mountpoint", sandbox.get(), Volume::RW)}));

  string dir = path::join(sandbox.get(), "dir");
  ASSERT_SOME(os::mkdir(dir));

  string directory = path::join(flags.work_dir, "sandbox");
  ASSERT_SOME(os::mkdir(directory));

  Future<Containerizer::LaunchResult> launch = containerizer->launch(
      containerId,
      createContainerConfig(None(), executor, directory),
      map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  Future<Option<ContainerTermination>> wait = containerizer->wait(containerId);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  ASSERT_TRUE(wait->get().has_status());
  EXPECT_WEXITSTATUS_EQ(0, wait->get().status());
}


// This test verifies that a file volume with an absolute host path
// and a relative container path is properly mounted in the container's
// mount namespace. The mount point will be created in the sandbox.
TEST_F(VolumeHostPathIsolatorTest, ROOT_FileVolumeFromHostSandboxMountPoint)
{
  string registry = path::join(sandbox.get(), "registry");
  AWAIT_READY(DockerArchive::create(registry, "test_image"));

  slave::Flags flags = CreateSlaveFlags();
  flags.isolation = "filesystem/linux,docker/runtime";
  flags.docker_registry = registry;
  flags.docker_store_dir = path::join(sandbox.get(), "store");
  flags.image_providers = "docker";

  Fetcher fetcher(flags);

  Try<MesosContainerizer*> create =
    MesosContainerizer::create(flags, true, &fetcher);

  ASSERT_SOME(create);

  Owned<Containerizer> containerizer(create.get());

  ContainerID containerId;
  containerId.set_value(id::UUID::random().toString());

  ExecutorInfo executor = createExecutorInfo(
      "test_executor",
      "test -f mountpoint/file.txt");

  string file = path::join(sandbox.get(), "file");
  ASSERT_SOME(os::touch(file));

  executor.mutable_container()->CopyFrom(createContainerInfo(
      "test_image",
      {createVolumeHostPath("mountpoint/file.txt", file, Volume::RW)}));

  string directory = path::join(flags.work_dir, "sandbox");
  ASSERT_SOME(os::mkdir(directory));

  Future<Containerizer::LaunchResult> launch = containerizer->launch(
      containerId,
      createContainerConfig(None(), executor, directory),
      map<string, string>(),
      None());

  AWAIT_ASSERT_EQ_FOR(
      Containerizer::LaunchResult::SUCCESS, launch, Seconds(60));

  Future<Option<ContainerTermination>> wait = containerizer->wait(containerId);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  ASSERT_TRUE(wait->get().has_status());
  EXPECT_WEXITSTATUS_EQ(0, wait->get().status());
}


// This test verifies that non-existing host paths under whitelist paths
// specified by the `host_path_volume_force_creation` agent flag can be
// properly created and mounted into container's mount namespace.
TEST_F(VolumeHostPathIsolatorTest, ROOT_VolumeFromHostForceCreation)
{
  const string imageName = "test-image";
  const string registryPath = path::join(sandbox.get(), "registry");
  AWAIT_READY(DockerArchive::create(registryPath, imageName));

  const string storePath = path::join(sandbox.get(), "store");
  const string mntPath = path::join(sandbox.get(), "mnt");
  const vector<string> whitelistedPaths = {
    path::join(mntPath, "whitelist-01"),
    path::join(mntPath, "whitelist-02")
  };

  slave::Flags flags = CreateSlaveFlags();
  flags.docker_registry = registryPath;
  flags.docker_store_dir = storePath;
  flags.image_providers = "docker";
  flags.isolation = "filesystem/linux,docker/runtime";
  flags.host_path_volume_force_creation = strings::join(
      HOST_PATH_WHITELIST_DELIM, whitelistedPaths);

  Fetcher fetcher(flags);

  const Try<MesosContainerizer*> create =
      MesosContainerizer::create(flags, true, &fetcher);

  ASSERT_SOME(create);

  Owned<Containerizer> containerizer(create.get());

  const string sandboxPath = path::join(flags.work_dir, "sandbox");
  ASSERT_SOME(os::mkdir(sandboxPath));

  // Specify non-existing paths.
  const vector<string> hostPaths = {
    path::join(mntPath, "whitelist-01", "a", "b", "c"),
    path::join(mntPath, "whitelist-02", "d", "e", "f")
  };

  // Ensure none of `hostPaths` exists.
  foreach (const string& hostPath, hostPaths) {
    ASSERT_FALSE(os::exists(hostPath));
  }

  ContainerID containerId;
  containerId.set_value(id::UUID::random().toString());

  ContainerInfo containerInfo = createContainerInfo(imageName, {
      createVolumeHostPath("/mnt/foo", hostPaths.front(), Volume::RW),
      createVolumeHostPath("/mnt/bar", hostPaths.back(), Volume::RW)});

  ExecutorInfo executor = createExecutorInfo(
      "test-executor",
      "test -d /mnt/foo -a -d /mnt/bar");
  executor.mutable_container()->CopyFrom(containerInfo);

  Future<Containerizer::LaunchResult> launch = containerizer->launch(
      containerId,
      createContainerConfig(None(), executor, sandboxPath),
      map<string, string>(),
      None());

  AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);

  Future<Option<ContainerTermination>> wait =
      containerizer->wait(containerId);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  ASSERT_TRUE(wait->get().has_status());
  EXPECT_WEXITSTATUS_EQ(0, wait->get().status());

  // Expect `hostPath` was created and is a directory.
  foreach (const string& hostPath, hostPaths) {
    EXPECT_TRUE(os::stat::isdir(hostPath));
  }
}


TEST_F(VolumeHostPathIsolatorTest, ROOT_MountPropagation)
{
  slave::Flags flags = CreateSlaveFlags();
  flags.isolation = "filesystem/linux";

  Fetcher fetcher(flags);

  Try<MesosContainerizer*> create =
    MesosContainerizer::create(flags, true, &fetcher);

  ASSERT_SOME(create);

  Owned<Containerizer> containerizer(create.get());

  ContainerID containerId;
  containerId.set_value(id::UUID::random().toString());

  string mountDirectory = path::join(flags.work_dir, "mount_directory");
  string mountPoint = path::join(mountDirectory, "mount_point");
  string filePath = path::join(mountPoint, "foo");

  ASSERT_SOME(os::mkdir(mountPoint));

  ExecutorInfo executor = createExecutorInfo(
      "test_executor",
      strings::format(
          "mount -t tmpfs tmpfs %s; touch %s",
          mountPoint,
          filePath).get());

  executor.mutable_container()->CopyFrom(createContainerInfo(
      None(),
      {createVolumeHostPath(
          mountDirectory,
          mountDirectory,
          Volume::RW,
          MountPropagation::BIDIRECTIONAL)}));

  string directory = path::join(flags.work_dir, "sandbox");
  ASSERT_SOME(os::mkdir(directory));

  Future<Containerizer::LaunchResult> launch = containerizer->launch(
      containerId,
      createContainerConfig(None(), executor, directory),
      map<string, string>(),
      None());

  AWAIT_READY(launch);

  Future<Option<ContainerTermination>> wait = containerizer->wait(containerId);

  AWAIT_READY(wait);
  ASSERT_SOME(wait.get());
  ASSERT_TRUE(wait->get().has_status());
  EXPECT_WEXITSTATUS_EQ(0, wait->get().status());

  // If the mount propagation has been setup properly, we should see
  // the file we touch'ed in 'mountPoint'.
  EXPECT_TRUE(os::exists(filePath));
}


class VolumeHostPathIsolatorMesosTest
  : public MesosTest,
    public WithParamInterface<ParamExecutorType> {};


INSTANTIATE_TEST_CASE_P(
    ExecutorType,
    VolumeHostPathIsolatorMesosTest,
    Values(
        ParamExecutorType::commandExecutor(),
        ParamExecutorType::defaultExecutor()),
    ParamExecutorType::Printer());


// This test verifies that the framework can launch a command task
// that specifies both container image and host volumes.
TEST_P(VolumeHostPathIsolatorMesosTest, ROOT_ChangeRootFilesystem)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  string registry = path::join(sandbox.get(), "registry");
  AWAIT_READY(DockerArchive::create(registry, "test_image"));

  slave::Flags flags = CreateSlaveFlags();
  flags.isolation = "filesystem/linux,docker/runtime";
  flags.docker_registry = registry;
  flags.docker_store_dir = path::join(sandbox.get(), "store");
  flags.image_providers = "docker";

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags);
  ASSERT_SOME(slave);

  MockScheduler sched;

  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      DEFAULT_CREDENTIAL);

  Future<FrameworkID> frameworkId;
  EXPECT_CALL(sched, registered(&driver, _, _))
    .WillOnce(FutureArg<1>(&frameworkId));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(offers);
  ASSERT_FALSE(offers->empty());

  const Offer& offer = offers.get()[0];

  // Preparing two volumes:
  // - host_path: dir1, container_path: /tmp
  // - host_path: dir2, container_path: relative_dir
  string dir1 = path::join(sandbox.get(), "dir1");
  ASSERT_SOME(os::mkdir(dir1));

  string testFile = path::join(dir1, "testfile");
  ASSERT_SOME(os::touch(testFile));

  string dir2 = path::join(sandbox.get(), "dir2");
  ASSERT_SOME(os::mkdir(dir2));

  TaskInfo task = createTask(
      offer.slave_id(),
      Resources::parse("cpus:0.1;mem:32;disk:32").get(),
      "test -f /tmp/testfile && test -d " +
      path::join(flags.sandbox_directory, "relative_dir"));

  task.mutable_container()->CopyFrom(createContainerInfo(
      "test_image",
      {createVolumeHostPath("/tmp", dir1, Volume::RW),
       createVolumeHostPath("relative_dir", dir2, Volume::RW)}));

  if (GetParam().isCommandExecutor()) {
    driver.acceptOffers(
        {offer.id()},
        {LAUNCH({task})});
  } else if (GetParam().isDefaultExecutor()) {
    ExecutorInfo executor;
    executor.mutable_executor_id()->set_value("default");
    executor.set_type(ExecutorInfo::DEFAULT);
    executor.mutable_framework_id()->CopyFrom(frameworkId.get());
    executor.mutable_resources()->CopyFrom(
        Resources::parse("cpus:0.1;mem:32;disk:32").get());

    TaskGroupInfo taskGroup;
    taskGroup.add_tasks()->CopyFrom(task);

    driver.acceptOffers(
        {offer.id()},
        {LAUNCH_GROUP(executor, taskGroup)});
  } else {
    FAIL() << "Unexpected executor type";
  }

  Future<TaskStatus> statusStarting;
  Future<TaskStatus> statusRunning;
  Future<TaskStatus> statusFinished;

  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusStarting))
    .WillOnce(FutureArg<1>(&statusRunning))
    .WillOnce(FutureArg<1>(&statusFinished));

  AWAIT_READY(statusStarting);
  EXPECT_EQ(TASK_STARTING, statusStarting->state());

  AWAIT_READY(statusRunning);
  EXPECT_EQ(TASK_RUNNING, statusRunning->state());

  AWAIT_READY(statusFinished);
  EXPECT_EQ(TASK_FINISHED, statusFinished->state());

  driver.stop();
  driver.join();
}

} // namespace tests {
} // namespace internal {
} // namespace mesos {