src/tests/containerizer/docker_volume_isolator_tests.cpp

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <utility>
#include <vector>

#include <mesos/mesos.hpp>

#include <process/gtest.hpp>

#include <stout/os.hpp>
#include <stout/path.hpp>

#include "linux/fs.hpp"

#include "slave/containerizer/mesos/containerizer.hpp"
#include "slave/containerizer/mesos/linux_launcher.hpp"

#include "slave/containerizer/mesos/isolators/docker/runtime.hpp"

#include "slave/containerizer/mesos/isolators/docker/volume/driver.hpp"
#include "slave/containerizer/mesos/isolators/docker/volume/isolator.hpp"
#include "slave/containerizer/mesos/isolators/docker/volume/paths.hpp"

#include "slave/containerizer/mesos/isolators/filesystem/linux.hpp"

#include "tests/flags.hpp"
#include "tests/mesos.hpp"
#include "tests/mock_docker.hpp"

namespace slave = mesos::internal::slave;

using process::Clock;
using process::Failure;
using process::Future;
using process::Owned;

using std::string;
using std::vector;

using mesos::internal::master::Master;

using mesos::internal::slave::DockerRuntimeIsolatorProcess;
using mesos::internal::slave::DockerVolumeIsolatorProcess;
using mesos::internal::slave::Fetcher;
using mesos::internal::slave::Launcher;
using mesos::internal::slave::LinuxFilesystemIsolatorProcess;
using mesos::internal::slave::LinuxLauncher;
using mesos::internal::slave::MesosContainerizer;
using mesos::internal::slave::MesosContainerizerProcess;
using mesos::internal::slave::Provisioner;
using mesos::internal::slave::Slave;

using mesos::master::detector::MasterDetector;

using mesos::slave::Isolator;

using slave::docker::volume::DriverClient;

using testing::DoAll;

namespace mesos {
namespace internal {
namespace tests {

class MockDockerVolumeDriverClient : public DriverClient
{
public:
  MockDockerVolumeDriverClient() {}

  ~MockDockerVolumeDriverClient() override {}

  MOCK_METHOD3(
      mount,
      Future<string>(
          const string& driver,
          const string& name,
          const hashmap<string, string>& options));

  MOCK_METHOD2(
      unmount,
      Future<Nothing>(
          const string& driver,
          const string& name));
};


class DockerVolumeIsolatorTest : public MesosTest
{
protected:
  void TearDown() override
  {
    // Try to remove any mounts under sandbox.
    if (::geteuid() == 0) {
      Try<Nothing> unmountAll = fs::unmountAll(sandbox.get(), MNT_DETACH);
      if (unmountAll.isError()) {
        LOG(ERROR) << "Failed to unmount '" << sandbox.get()
                   << "': " << unmountAll.error();
        return;
      }
    }

    MesosTest::TearDown();
  }

  Volume createDockerVolume(
      const string& driver,
      const string& name,
      const string& containerPath,
      const Option<hashmap<string, string>>& options = None())
  {
    Volume volume;
    volume.set_mode(Volume::RW);
    volume.set_container_path(containerPath);

    Volume::Source* source = volume.mutable_source();
    source->set_type(Volume::Source::DOCKER_VOLUME);

    Volume::Source::DockerVolume* docker = source->mutable_docker_volume();
    docker->set_driver(driver);
    docker->set_name(name);

    Parameters parameters;

    if (options.isSome()) {
      foreachpair (const string& key, const string& value, options.get()) {
        Parameter* parameter = parameters.add_parameter();
        parameter->set_key(key);
        parameter->set_value(value);
      }

      docker->mutable_driver_options()->CopyFrom(parameters);
    }

    return volume;
  }

  // This helper creates a MesosContainerizer instance that uses the
  // LinuxFilesystemIsolator and DockerVolumeIsolator.
  Try<Owned<MesosContainerizer>> createContainerizer(
      const slave::Flags& flags,
      const Owned<DriverClient>& mockClient)
  {
    fetcher.reset(new Fetcher(flags));

    Try<Isolator*> linuxIsolator_ =
      LinuxFilesystemIsolatorProcess::create(flags);

    if (linuxIsolator_.isError()) {
      return Error(
          "Failed to create LinuxFilesystemIsolator: " +
          linuxIsolator_.error());
    }

    Owned<Isolator> linuxIsolator(linuxIsolator_.get());

    Try<Isolator*> runtimeIsolator_ =
      DockerRuntimeIsolatorProcess::create(flags);

    if (runtimeIsolator_.isError()) {
      return Error(
          "Failed to create DockerRuntimeIsolator: " +
          runtimeIsolator_.error());
    }

    Owned<Isolator> runtimeIsolator(runtimeIsolator_.get());

    Try<Isolator*> volumeIsolator_ =
      DockerVolumeIsolatorProcess::_create(flags, mockClient);

    if (volumeIsolator_.isError()) {
      return Error(
          "Failed to create DockerVolumeIsolator: " +
          volumeIsolator_.error());
    }

    Owned<Isolator> volumeIsolator(volumeIsolator_.get());

    Try<Launcher*> launcher_ = LinuxLauncher::create(flags);
    if (launcher_.isError()) {
      return Error("Failed to create LinuxLauncher: " + launcher_.error());
    }

    Owned<Launcher> launcher(launcher_.get());

    Try<Owned<Provisioner>> provisioner = Provisioner::create(flags);
    if (provisioner.isError()) {
      return Error("Failed to create provisioner: " + provisioner.error());
    }

    Try<MesosContainerizer*> containerizer = MesosContainerizer::create(
        flags,
        true,
        fetcher.get(),
        nullptr,
        std::move(launcher),
        provisioner->share(),
        {std::move(linuxIsolator),
         std::move(runtimeIsolator),
         std::move(volumeIsolator)});

    if (containerizer.isError()) {
      return Error("Failed to create containerizer: " + containerizer.error());
    }

    return Owned<MesosContainerizer>(containerizer.get());
  }

private:
  Owned<Fetcher> fetcher;
};


// This test verifies that multiple docker volumes with both absolute
// path and relative path are properly mounted to a container without
// rootfs, and launches a command task that reads files from the
// mounted docker volumes.
TEST_F(DockerVolumeIsolatorTest, ROOT_CommandTaskNoRootfsWithVolumes)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  slave::Flags flags = CreateSlaveFlags();

  MockDockerVolumeDriverClient* mockClient = new MockDockerVolumeDriverClient;

  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer(flags, Owned<DriverClient>(mockClient));

  ASSERT_SOME(containerizer);

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(
      detector.get(),
      containerizer->get(),
      flags);

  ASSERT_SOME(slave);

  MockScheduler sched;

  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      DEFAULT_CREDENTIAL);

  Future<FrameworkID> frameworkId;
  EXPECT_CALL(sched, registered(&driver, _, _))
    .WillOnce(FutureArg<1>(&frameworkId));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(frameworkId);

  AWAIT_READY(offers);
  ASSERT_FALSE(offers->empty());

  const Offer& offer = offers.get()[0];

  const string key = "iops";
  const string value = "150";

  hashmap<string, string> options = {{key, value}};

  // Create a volume with relative path.
  const string driver1 = "driver1";
  const string name1 = "name1";
  const string containerPath1 = "tmp/foo1";

  Volume volume1 = createDockerVolume(driver1, name1, containerPath1, options);

  // Create a volume with absolute path.
  const string driver2 = "driver2";
  const string name2 = "name2";

  // Make sure the absolute path exist.
  const string containerPath2 = path::join(os::getcwd(), "foo2");
  ASSERT_SOME(os::mkdir(containerPath2));

  Volume volume2 = createDockerVolume(driver2, name2, containerPath2);

  TaskInfo task = createTask(
      offer.slave_id(),
      offer.resources(),
      "test -f " + containerPath1 + "/file1 && "
      "test -f " + containerPath2 + "/file2;");

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.add_volumes()->CopyFrom(volume1);
  containerInfo.add_volumes()->CopyFrom(volume2);

  task.mutable_container()->CopyFrom(containerInfo);

  // Create mount point for volume1.
  const string mountPoint1 = path::join(os::getcwd(), "volume1");
  ASSERT_SOME(os::mkdir(mountPoint1));
  ASSERT_SOME(os::touch(path::join(mountPoint1, "file1")));

  // Create mount point for volume2.
  const string mountPoint2 = path::join(os::getcwd(), "volume2");
  ASSERT_SOME(os::mkdir(mountPoint2));
  ASSERT_SOME(os::touch(path::join(mountPoint2, "file2")));

  Future<string> mount1Name;
  Future<string> mount2Name;
  Future<hashmap<string, string>> mount1Options;

  EXPECT_CALL(*mockClient, mount(driver1, _, _))
    .WillOnce(DoAll(FutureArg<1>(&mount1Name),
                    FutureArg<2>(&mount1Options),
                    Return(mountPoint1)));

  EXPECT_CALL(*mockClient, mount(driver2, _, _))
    .WillOnce(DoAll(FutureArg<1>(&mount2Name),
                    Return(mountPoint2)));

  Future<string> unmount1Name;
  Future<string> unmount2Name;

  EXPECT_CALL(*mockClient, unmount(driver1, _))
    .WillOnce(DoAll(FutureArg<1>(&unmount1Name),
                    Return(Nothing())));

  EXPECT_CALL(*mockClient, unmount(driver2, _))
    .WillOnce(DoAll(FutureArg<1>(&unmount2Name),
                    Return(Nothing())));

  Future<TaskStatus> statusStarting;
  Future<TaskStatus> statusRunning;
  Future<TaskStatus> statusFinished;

  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusStarting))
    .WillOnce(FutureArg<1>(&statusRunning))
    .WillOnce(FutureArg<1>(&statusFinished));

  driver.launchTasks(offer.id(), {task});

  AWAIT_READY(statusStarting);
  EXPECT_EQ(TASK_STARTING, statusStarting->state());

  AWAIT_READY(statusRunning);
  EXPECT_EQ(TASK_RUNNING, statusRunning->state());

  // Make sure the docker volume mount parameters are same with the
  // parameters in `containerInfo`.
  AWAIT_EXPECT_EQ(name1, mount1Name);
  AWAIT_EXPECT_EQ(name2, mount2Name);

  AWAIT_READY(mount1Options);
  EXPECT_SOME_EQ(value, mount1Options->get(key));

  AWAIT_READY(statusFinished);
  EXPECT_EQ(TASK_FINISHED, statusFinished->state());

  // Make sure the docker volume unmount parameters are same with
  // the parameters in `containerInfo`.
  AWAIT_EXPECT_EQ(name1, unmount1Name);
  AWAIT_EXPECT_EQ(name2, unmount2Name);

  driver.stop();
  driver.join();
}


// This test verifies that multiple docker volumes with the same
// driver and name cannot be mounted to the same container. If that
// happens, the task will fail.
TEST_F(DockerVolumeIsolatorTest, ROOT_CommandTaskNoRootfsFailedWithSameVolumes)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  slave::Flags flags = CreateSlaveFlags();

  MockDockerVolumeDriverClient* mockClient = new MockDockerVolumeDriverClient;

  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer(flags, Owned<DriverClient>(mockClient));

  ASSERT_SOME(containerizer);

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(
      detector.get(),
      containerizer->get(),
      flags);

  ASSERT_SOME(slave);

  MockScheduler sched;

  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      DEFAULT_CREDENTIAL);

  Future<FrameworkID> frameworkId;
  EXPECT_CALL(sched, registered(&driver, _, _))
    .WillOnce(FutureArg<1>(&frameworkId));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(frameworkId);

  AWAIT_READY(offers);
  ASSERT_FALSE(offers->empty());

  const Offer& offer = offers.get()[0];

  const string key = "iops";
  const string value = "150";

  hashmap<string, string> options = {{key, value}};

  // Create a volume with relative path.
  const string driver1 = "driver1";
  const string name1 = "name1";
  const string containerPath1 = "tmp/foo1";

  Volume volume1 = createDockerVolume(driver1, name1, containerPath1, options);

  // Create a volume with absolute path and make sure the absolute
  // path exist. Please note that volume1 and volume2 will be created
  // with same volume driver and name, this will cause task failed
  // when mounting same mount point to one container.
  const string containerPath2 = path::join(os::getcwd(), "foo2");
  ASSERT_SOME(os::mkdir(containerPath2));

  Volume volume2 = createDockerVolume(driver1, name1, containerPath2);

  TaskInfo task = createTask(
      offer.slave_id(),
      offer.resources(),
      "test -f " + containerPath1 + "/file1 && "
      "test -f " + containerPath2 + "/file2;");

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.add_volumes()->CopyFrom(volume1);
  containerInfo.add_volumes()->CopyFrom(volume2);

  task.mutable_container()->CopyFrom(containerInfo);

  Future<TaskStatus> statusFailed;
  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusFailed));

  driver.launchTasks(offer.id(), {task});

  AWAIT_READY(statusFailed);
  EXPECT_EQ(task.task_id(), statusFailed->task_id());
  EXPECT_EQ(TASK_FAILED, statusFailed->state());

  driver.stop();
  driver.join();
}


// This test verifies that the docker volumes are properly recovered
// during slave recovery.
TEST_F(DockerVolumeIsolatorTest, ROOT_CommandTaskNoRootfsSlaveRecovery)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  slave::Flags flags = CreateSlaveFlags();

  MockDockerVolumeDriverClient* mockClient = new MockDockerVolumeDriverClient;

  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer(flags, Owned<DriverClient>(mockClient));

  ASSERT_SOME(containerizer);

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(
      detector.get(),
      containerizer->get(),
      flags);

  ASSERT_SOME(slave);

  // Enable checkpointing for the framework.
  FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
  frameworkInfo.set_checkpoint(true);

  MockScheduler sched;

  MesosSchedulerDriver driver(
      &sched,
      frameworkInfo,
      master.get()->pid,
      DEFAULT_CREDENTIAL);

  Future<FrameworkID> frameworkId;
  EXPECT_CALL(sched, registered(&driver, _, _))
    .WillOnce(FutureArg<1>(&frameworkId));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(frameworkId);

  AWAIT_READY(offers);
  ASSERT_FALSE(offers->empty());

  const Offer& offer = offers.get()[0];

  const string key = "iops";
  const string value = "150";

  hashmap<string, string> options = {{key, value}};

  // Create a volume with relative path.
  const string driver1 = "driver1";
  const string name1 = "name1";
  const string containerPath1 = "tmp/foo1";

  Volume volume1 = createDockerVolume(driver1, name1, containerPath1, options);

  // Create a volume with absolute path.
  const string driver2 = "driver2";
  const string name2 = "name2";

  // Make sure the absolute path exist.
  const string containerPath2 = path::join(os::getcwd(), "foo2");
  ASSERT_SOME(os::mkdir(containerPath2));

  Volume volume2 = createDockerVolume(driver2, name2, containerPath2);

  TaskInfo task = createTask(
      offer.slave_id(),
      offer.resources(),
      "while true; do test -f " + containerPath1 + "/file1 && "
      "test -f " + containerPath2 + "/file2; done");

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.add_volumes()->CopyFrom(volume1);
  containerInfo.add_volumes()->CopyFrom(volume2);

  task.mutable_container()->CopyFrom(containerInfo);

  // Create mount point for volume1.
  const string mountPoint1 = path::join(os::getcwd(), "volume1");
  ASSERT_SOME(os::mkdir(mountPoint1));
  ASSERT_SOME(os::touch(path::join(mountPoint1, "file1")));

  // Create mount point for volume2.
  const string mountPoint2 = path::join(os::getcwd(), "volume2");
  ASSERT_SOME(os::mkdir(mountPoint2));
  ASSERT_SOME(os::touch(path::join(mountPoint2, "file2")));

  Future<string> mount1Name;
  Future<string> mount2Name;
  Future<hashmap<string, string>> mount1Options;

  EXPECT_CALL(*mockClient, mount(driver1, _, _))
    .WillOnce(DoAll(FutureArg<1>(&mount1Name),
                    FutureArg<2>(&mount1Options),
                    Return(mountPoint1)));

  EXPECT_CALL(*mockClient, mount(driver2, _, _))
    .WillOnce(DoAll(FutureArg<1>(&mount2Name),
                    Return(mountPoint2)));

  Future<TaskStatus> statusStarting;
  Future<TaskStatus> statusRunning;

  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusStarting))
    .WillOnce(FutureArg<1>(&statusRunning));

  Future<Nothing> ack1 =
    FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement);

  Future<Nothing> ack2 =
    FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement);

  driver.launchTasks(offer.id(), {task});

  AWAIT_READY(statusStarting);
  EXPECT_EQ(TASK_STARTING, statusStarting->state());

  // Wait for the ACK to be checkpointed.
  AWAIT_READY(ack1);

  AWAIT_READY(statusRunning);
  EXPECT_EQ(TASK_RUNNING, statusRunning->state());

  // Wait for the ACK to be checkpointed.
  AWAIT_READY(ack2);

  // Stop the slave after TASK_RUNNING is received.
  slave.get()->terminate();

  // Set up so we can wait until the new slave updates the container's
  // resources (this occurs after the executor has reregistered).
  Future<Nothing> update =
    FUTURE_DISPATCH(_, &MesosContainerizerProcess::update);

  mockClient = new MockDockerVolumeDriverClient;

  containerizer = createContainerizer(
      flags, Owned<DriverClient>(mockClient));

  ASSERT_SOME(containerizer);

  Future<SlaveReregisteredMessage> reregistered =
      FUTURE_PROTOBUF(SlaveReregisteredMessage(), master.get()->pid, _);

  Future<string> unmount1Name;
  Future<string> unmount2Name;

  EXPECT_CALL(*mockClient, unmount(driver1, _))
    .WillOnce(DoAll(FutureArg<1>(&unmount1Name),
                    Return(Nothing())));

  EXPECT_CALL(*mockClient, unmount(driver2, _))
    .WillOnce(DoAll(FutureArg<1>(&unmount2Name),
                    Return(Nothing())));

  // Use the same flags.
  slave = StartSlave(detector.get(), containerizer->get(), flags);
  ASSERT_SOME(slave);

  AWAIT_READY(reregistered);

  // Wait until the containerizer is updated.
  AWAIT_READY(update);

  Future<hashset<ContainerID>> containers = containerizer.get()->containers();
  AWAIT_READY(containers);
  ASSERT_EQ(1u, containers->size());

  ContainerID containerId = *(containers->begin());

  Future<TaskStatus> statusKilled;
  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusKilled));

  // Kill the task.
  driver.killTask(task.task_id());

  AWAIT_READY(statusKilled);
  EXPECT_EQ(TASK_KILLED, statusKilled->state());

  // Make sure the docker volume unmount parameters are same with
  // the parameters in `containerInfo`.
  AWAIT_EXPECT_EQ(name1, unmount1Name);
  AWAIT_EXPECT_EQ(name2, unmount2Name);

  driver.stop();
  driver.join();
}


// This test verifies that agent can recover properly even there is
// an orphan container's docker volumes checkpoint file is empty which
// could happen in a real environment if the agent is hard rebooted
// after the file is created but before the data is synced on disk.
TEST_F(DockerVolumeIsolatorTest, ROOT_EmptyCheckpointFileSlaveRecovery)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  slave::Flags flags = CreateSlaveFlags();

  MockDockerVolumeDriverClient* mockClient = new MockDockerVolumeDriverClient;

  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer(flags, Owned<DriverClient>(mockClient));

  ASSERT_SOME(containerizer);

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(
      detector.get(),
      containerizer->get(),
      flags);

  ASSERT_SOME(slave);

  FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;

  MockScheduler sched;

  MesosSchedulerDriver driver(
      &sched,
      frameworkInfo,
      master.get()->pid,
      DEFAULT_CREDENTIAL);

  Future<FrameworkID> frameworkId;
  EXPECT_CALL(sched, registered(&driver, _, _))
    .WillOnce(FutureArg<1>(&frameworkId));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(frameworkId);

  AWAIT_READY(offers);
  ASSERT_FALSE(offers->empty());

  const Offer& offer = offers.get()[0];

  // Create a volume with relative path.
  const string driver1 = "driver1";
  const string name1 = "name1";
  const string containerPath1 = "tmp/foo1";

  Volume volume1 = createDockerVolume(driver1, name1, containerPath1);

  TaskInfo task = createTask(
      offer.slave_id(),
      offer.resources(),
      "while true; do test -f " + containerPath1 + "/file1; done");

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.add_volumes()->CopyFrom(volume1);

  task.mutable_container()->CopyFrom(containerInfo);

  // Create mount point for volume1.
  const string mountPoint1 = path::join(os::getcwd(), "volume1");
  ASSERT_SOME(os::mkdir(mountPoint1));
  ASSERT_SOME(os::touch(path::join(mountPoint1, "file1")));

  Future<string> mount1Name;

  EXPECT_CALL(*mockClient, mount(driver1, _, _))
    .WillOnce(DoAll(FutureArg<1>(&mount1Name),
                    Return(mountPoint1)));

  Future<TaskStatus> statusStarting;
  Future<TaskStatus> statusRunning;

  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusStarting))
    .WillOnce(FutureArg<1>(&statusRunning))
    .WillRepeatedly(Return()); // Ignore subsequent updates.

  Future<Nothing> ack1 =
    FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement);

  Future<Nothing> ack2 =
    FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement);

  driver.launchTasks(offer.id(), {task});

  AWAIT_READY(statusStarting);
  EXPECT_EQ(TASK_STARTING, statusStarting->state());

  // Wait for the ACK to be checkpointed.
  AWAIT_READY(ack1);

  AWAIT_READY(statusRunning);
  EXPECT_EQ(TASK_RUNNING, statusRunning->state());

  // Wait for the ACK to be checkpointed.
  AWAIT_READY(ack2);

  Future<hashset<ContainerID>> containers = containerizer.get()->containers();
  AWAIT_READY(containers);
  ASSERT_EQ(1u, containers->size());

  ContainerID containerId = *(containers->begin());

  // Stop the slave after TASK_RUNNING is received.
  slave.get()->terminate();

  // Make the docker volumes checkpoint file empty. This is to simulate the
  // case that the agent is hard rebooted after the file is created but before
  // the data is synced on disk, i.e., the file is left empty on disk.
  const string volumesPath = slave::docker::volume::paths::getVolumesPath(
      flags.docker_volume_checkpoint_dir,
      containerId.value());

  ASSERT_SOME(os::write(volumesPath, ""));

  mockClient = new MockDockerVolumeDriverClient;

  containerizer = createContainerizer(flags, Owned<DriverClient>(mockClient));

  ASSERT_SOME(containerizer);

  Future<SlaveReregisteredMessage> reregistered =
    FUTURE_PROTOBUF(SlaveReregisteredMessage(), master.get()->pid, _);

  // Use the same flags.
  slave = StartSlave(detector.get(), containerizer->get(), flags);
  ASSERT_SOME(slave);

  AWAIT_READY(reregistered);

  driver.stop();
  driver.join();
}


// This test verifies that a single docker volumes can be used by
// multiple containers, and the docker volume isolator will mount
// the single volume to multiple containers when running tasks and
// the docker volume isolator will call unmount only once when cleanup
// the last container that is using the volume.
TEST_F(DockerVolumeIsolatorTest,
       ROOT_CommandTaskNoRootfsSingleVolumeMultipleContainers)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  slave::Flags flags = CreateSlaveFlags();

  MockDockerVolumeDriverClient* mockClient = new MockDockerVolumeDriverClient;

  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer(flags, Owned<DriverClient>(mockClient));

  ASSERT_SOME(containerizer);

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(
      detector.get(),
      containerizer->get(),
      flags);

  ASSERT_SOME(slave);

  MockScheduler sched;

  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      DEFAULT_CREDENTIAL);

  Future<FrameworkID> frameworkId;
  EXPECT_CALL(sched, registered(&driver, _, _))
    .WillOnce(FutureArg<1>(&frameworkId));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(frameworkId);

  AWAIT_READY(offers);
  ASSERT_FALSE(offers->empty());

  const Offer& offer = offers.get()[0];

  // Create a volume with relative path and share the volume with
  // two different containers.
  const string driver1 = "driver1";
  const string name1 = "name1";
  const string containerPath1 = "tmp/foo";

  Volume volume1 = createDockerVolume(driver1, name1, containerPath1);

  TaskInfo task1 = createTask(
      offer.slave_id(),
      Resources::parse("cpus:1;mem:64").get(),
      "while true; do test -f " + containerPath1 + "/file; done");

  ContainerInfo containerInfo1;
  containerInfo1.set_type(ContainerInfo::MESOS);
  containerInfo1.add_volumes()->CopyFrom(volume1);

  task1.mutable_container()->CopyFrom(containerInfo1);

  // Create mount point for volume.
  const string mountPoint1 = path::join(os::getcwd(), "volume");
  ASSERT_SOME(os::mkdir(mountPoint1));
  ASSERT_SOME(os::touch(path::join(mountPoint1, "file")));

  TaskInfo task2 = createTask(
      offer.slave_id(),
      Resources::parse("cpus:1;mem:64").get(),
      "while true; do test -f " + containerPath1 + "/file; done");

  ContainerInfo containerInfo2;
  containerInfo2.set_type(ContainerInfo::MESOS);
  containerInfo2.add_volumes()->CopyFrom(volume1);

  task2.mutable_container()->CopyFrom(containerInfo2);

  // The mount operation will be called multiple times as there are
  // two containers using the same volume.
  EXPECT_CALL(*mockClient, mount(driver1, _, _))
    .WillRepeatedly(Return(mountPoint1));

  // Expect the unmount was called only once because two containers
  // sharing one volume.
  EXPECT_CALL(*mockClient, unmount(driver1, _))
    .WillOnce(Return(Nothing()));

  Future<TaskStatus> statusStarting1;
  Future<TaskStatus> statusRunning1;
  Future<TaskStatus> statusKilled1;
  Future<TaskStatus> statusStarting2;
  Future<TaskStatus> statusRunning2;
  Future<TaskStatus> statusKilled2;

  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusStarting1))
    .WillOnce(FutureArg<1>(&statusStarting2))
    .WillOnce(FutureArg<1>(&statusRunning1))
    .WillOnce(FutureArg<1>(&statusRunning2))
    .WillOnce(FutureArg<1>(&statusKilled1))
    .WillOnce(FutureArg<1>(&statusKilled2));

  driver.launchTasks(offer.id(), {task1, task2});

  // TASK_STARTING and TASK_RUNNING updates might arrive interleaved,
  // so we only check the first and the last where the values are known.
  AWAIT_READY(statusStarting1);
  EXPECT_EQ(TASK_STARTING, statusStarting1->state());

  AWAIT_READY(statusRunning2);
  EXPECT_EQ(TASK_RUNNING, statusRunning2->state());

  // Kill both of the tasks.
  driver.killTask(task1.task_id());
  driver.killTask(task2.task_id());

  AWAIT_READY(statusKilled1);
  EXPECT_EQ(TASK_KILLED, statusKilled1->state());

  AWAIT_READY(statusKilled2);
  EXPECT_EQ(TASK_KILLED, statusKilled2->state());

  driver.stop();
  driver.join();
}


// This test verifies that a docker volume with absolute path can
// be properly mounted to a container with rootfs, and launches a
// command task that reads files from the mounted docker volume.
TEST_F(DockerVolumeIsolatorTest,
       ROOT_INTERNET_CURL_CommandTaskRootfsWithAbsolutePathVolume)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  slave::Flags flags = CreateSlaveFlags();
  flags.isolation = "docker/volume,docker/runtime,filesystem/linux";
  flags.image_providers = "docker";

  MockDockerVolumeDriverClient* mockClient = new MockDockerVolumeDriverClient;

  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer(flags, Owned<DriverClient>(mockClient));

  ASSERT_SOME(containerizer);

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(
      detector.get(),
      containerizer->get(),
      flags);

  ASSERT_SOME(slave);

  MockScheduler sched;

  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      DEFAULT_CREDENTIAL);

  Future<FrameworkID> frameworkId;
  EXPECT_CALL(sched, registered(&driver, _, _))
    .WillOnce(FutureArg<1>(&frameworkId));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(frameworkId);

  AWAIT_READY(offers);
  ASSERT_FALSE(offers->empty());

  const Offer& offer = offers.get()[0];

  // Create a volume with absolute path.
  const string volumeDriver = "driver";
  const string name = "name";

  const string containerPath = path::join(os::getcwd(), "foo");

  Volume volume = createDockerVolume(volumeDriver, name, containerPath);

  // NOTE: We use a non-shell command here because 'sh' might not be
  // in the PATH. 'alpine' does not specify env PATH in the image. On
  // some linux distribution, '/bin' is not in the PATH by default.
  CommandInfo command;
  command.set_shell(false);
  command.set_value("/usr/bin/test");
  command.add_arguments("test");
  command.add_arguments("-f");
  command.add_arguments(containerPath + "/file");

  TaskInfo task = createTask(
      offer.slave_id(),
      offer.resources(),
      command);

  Image image;
  image.set_type(Image::DOCKER);
  image.mutable_docker()->set_name("alpine");

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.add_volumes()->CopyFrom(volume);

  containerInfo.mutable_mesos()->mutable_image()->CopyFrom(image);

  task.mutable_container()->CopyFrom(containerInfo);

  // Create mount point for volume.
  const string mountPoint = path::join(os::getcwd(), "volume");
  ASSERT_SOME(os::mkdir(mountPoint));
  ASSERT_SOME(os::touch(path::join(mountPoint, "file")));

  Future<string> mountName;

  EXPECT_CALL(*mockClient, mount(volumeDriver, _, _))
    .WillOnce(DoAll(FutureArg<1>(&mountName),
                    Return(mountPoint)));

  Future<string> unmountName;

  EXPECT_CALL(*mockClient, unmount(volumeDriver, _))
    .WillOnce(DoAll(FutureArg<1>(&unmountName),
                    Return(Nothing())));

  Future<TaskStatus> statusStarting;
  Future<TaskStatus> statusRunning;
  Future<TaskStatus> statusFinished;

  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusStarting))
    .WillOnce(FutureArg<1>(&statusRunning))
    .WillOnce(FutureArg<1>(&statusFinished));

  driver.launchTasks(offer.id(), {task});

  AWAIT_READY(statusStarting);
  EXPECT_EQ(TASK_STARTING, statusStarting->state());

  AWAIT_READY(statusRunning);
  EXPECT_EQ(TASK_RUNNING, statusRunning->state());

  // Make sure the docker volume mount parameters are same with the
  // parameters in `containerInfo`.
  AWAIT_EXPECT_EQ(name, mountName);

  AWAIT_READY(statusFinished);
  EXPECT_EQ(TASK_FINISHED, statusFinished->state());

  // Make sure the docker volume unmount parameters are same with
  // the parameters in `containerInfo`.
  AWAIT_EXPECT_EQ(name, unmountName);

  driver.stop();
  driver.join();
}


// This test verifies that a docker volume with relative path can
// be properly mounted to a container with rootfs, and launches a
// command task that reads files from the mounted docker volume.
TEST_F(DockerVolumeIsolatorTest,
       ROOT_INTERNET_CURL_CommandTaskRootfsWithRelativeVolume)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  slave::Flags flags = CreateSlaveFlags();
  flags.isolation = "docker/volume,docker/runtime,filesystem/linux";
  flags.image_providers = "docker";

  MockDockerVolumeDriverClient* mockClient = new MockDockerVolumeDriverClient;

  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer(flags, Owned<DriverClient>(mockClient));

  ASSERT_SOME(containerizer);

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(
      detector.get(),
      containerizer->get(),
      flags);

  ASSERT_SOME(slave);

  MockScheduler sched;

  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      DEFAULT_CREDENTIAL);

  Future<FrameworkID> frameworkId;
  EXPECT_CALL(sched, registered(&driver, _, _))
    .WillOnce(FutureArg<1>(&frameworkId));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(frameworkId);

  AWAIT_READY(offers);
  ASSERT_FALSE(offers->empty());

  const Offer& offer = offers.get()[0];

  const string key = "iops";
  const string value = "150";

  hashmap<string, string> options = {{key, value}};

  // Create a volume with relative path.
  const string volumeDriver = "driver";
  const string name = "name";
  const string containerPath = "tmp/foo";

  Volume volume = createDockerVolume(
      volumeDriver, name, containerPath, options);

  // NOTE: We use a non-shell command here because 'sh' might not be
  // in the PATH. 'alpine' does not specify env PATH in the image. On
  // some linux distribution, '/bin' is not in the PATH by default.
  CommandInfo command;
  command.set_shell(false);
  command.set_value("/usr/bin/test");
  command.add_arguments("test");
  command.add_arguments("-f");
  command.add_arguments(containerPath + "/file");

  TaskInfo task = createTask(
      offer.slave_id(),
      offer.resources(),
      command);

  Image image;
  image.set_type(Image::DOCKER);
  image.mutable_docker()->set_name("alpine");

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.add_volumes()->CopyFrom(volume);

  containerInfo.mutable_mesos()->mutable_image()->CopyFrom(image);

  task.mutable_container()->CopyFrom(containerInfo);

  // Create mount point for volume.
  const string mountPoint = path::join(os::getcwd(), "volume");
  ASSERT_SOME(os::mkdir(mountPoint));
  ASSERT_SOME(os::touch(path::join(mountPoint, "file")));

  Future<string> mountName;
  Future<hashmap<string, string>> mountOptions;

  EXPECT_CALL(*mockClient, mount(volumeDriver, _, _))
    .WillOnce(DoAll(FutureArg<1>(&mountName),
                    FutureArg<2>(&mountOptions),
                    Return(mountPoint)));

  Future<string> unmountName;

  EXPECT_CALL(*mockClient, unmount(volumeDriver, _))
    .WillOnce(DoAll(FutureArg<1>(&unmountName),
                    Return(Nothing())));

  Future<TaskStatus> statusStarting;
  Future<TaskStatus> statusRunning;
  Future<TaskStatus> statusFinished;

  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusStarting))
    .WillOnce(FutureArg<1>(&statusRunning))
    .WillOnce(FutureArg<1>(&statusFinished));

  driver.launchTasks(offer.id(), {task});

  AWAIT_READY(statusStarting);
  EXPECT_EQ(TASK_STARTING, statusStarting->state());

  AWAIT_READY(statusRunning);
  EXPECT_EQ(TASK_RUNNING, statusRunning->state());

  // Make sure the docker volume mount parameters are same with the
  // parameters in `containerInfo`.
  AWAIT_EXPECT_EQ(name, mountName);

  AWAIT_READY(mountOptions);
  EXPECT_SOME_EQ(value, mountOptions->get(key));

  AWAIT_READY(statusFinished);
  EXPECT_EQ(TASK_FINISHED, statusFinished->state());

  // Make sure the docker volume unmount parameters are same with
  // the parameters in `containerInfo`.
  AWAIT_EXPECT_EQ(name, unmountName);

  driver.stop();
  driver.join();
}


// This test verifies that a docker volume with absolute path can
// be properly mounted to a container with rootfs, and launches a
// command task that is running as a non-root user and is able to
// write files from the mounted docker volume.
TEST_F(DockerVolumeIsolatorTest,
       ROOT_INTERNET_CURL_UNPRIVILEGED_USER_CommandTaskRootfs)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  slave::Flags flags = CreateSlaveFlags();
  flags.isolation = "docker/volume,docker/runtime,filesystem/linux";
  flags.image_providers = "docker";
  flags.docker_volume_chown = true;

  MockDockerVolumeDriverClient* mockClient = new MockDockerVolumeDriverClient;

  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer(flags, Owned<DriverClient>(mockClient));

  ASSERT_SOME(containerizer);

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(
      detector.get(),
      containerizer->get(),
      flags);

  ASSERT_SOME(slave);

  MockScheduler sched;

  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      DEFAULT_CREDENTIAL);

  Future<FrameworkID> frameworkId;
  EXPECT_CALL(sched, registered(&driver, _, _))
    .WillOnce(FutureArg<1>(&frameworkId));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(frameworkId);

  AWAIT_READY(offers);
  ASSERT_FALSE(offers->empty());

  const Offer& offer = offers.get()[0];

  // Create a volume with absolute path.
  const string volumeDriver = "driver";
  const string name = "name";

  const string containerPath = path::join(os::getcwd(), "foo");

  Volume volume = createDockerVolume(volumeDriver, name, containerPath);

  Option<string> user = os::getenv("SUDO_USER");
  ASSERT_SOME(user);

  // NOTE: We use a non-shell command here because 'sh' might not be
  // in the PATH. 'alpine' does not specify env PATH in the image. On
  // some linux distribution, '/bin' is not in the PATH by default.
  CommandInfo command;
  command.set_shell(false);
  command.set_value("/usr/bin/test");
  command.add_arguments("test");
  command.add_arguments("-w");
  command.add_arguments(containerPath);

  command.set_user(user.get());

  TaskInfo task = createTask(
      offer.slave_id(),
      offer.resources(),
      command);

  Image image;
  image.set_type(Image::DOCKER);
  image.mutable_docker()->set_name("alpine");

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.add_volumes()->CopyFrom(volume);

  containerInfo.mutable_mesos()->mutable_image()->CopyFrom(image);

  task.mutable_container()->CopyFrom(containerInfo);

  // Create mount point for volume.
  const string mountPoint = path::join(os::getcwd(), "volume");
  ASSERT_SOME(os::mkdir(mountPoint));

  Future<string> mountName;

  EXPECT_CALL(*mockClient, mount(volumeDriver, _, _))
    .WillOnce(DoAll(FutureArg<1>(&mountName),
                    Return(mountPoint)));

  Future<string> unmountName;

  EXPECT_CALL(*mockClient, unmount(volumeDriver, _))
    .WillOnce(DoAll(FutureArg<1>(&unmountName),
                    Return(Nothing())));

  Future<TaskStatus> statusStarting;
  Future<TaskStatus> statusRunning;
  Future<TaskStatus> statusFinished;

  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusStarting))
    .WillOnce(FutureArg<1>(&statusRunning))
    .WillOnce(FutureArg<1>(&statusFinished));

  driver.launchTasks(offer.id(), {task});

  AWAIT_READY(statusStarting);
  EXPECT_EQ(TASK_STARTING, statusStarting->state());

  AWAIT_READY(statusRunning);
  EXPECT_EQ(TASK_RUNNING, statusRunning->state());

  AWAIT_EXPECT_EQ(name, mountName);

  AWAIT_READY(statusFinished);
  EXPECT_EQ(TASK_FINISHED, statusFinished->state());

  AWAIT_EXPECT_EQ(name, unmountName);

  driver.stop();
  driver.join();
}


// This test verifies that a container launched without
// a rootfs cannot write to a read-only docker volume.
TEST_F(DockerVolumeIsolatorTest, ROOT_CommandTaskNoRootfsWithReadOnlyVolume)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  slave::Flags flags = CreateSlaveFlags();

  MockDockerVolumeDriverClient* mockClient = new MockDockerVolumeDriverClient;

  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer(flags, Owned<DriverClient>(mockClient));

  ASSERT_SOME(containerizer);

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(
      detector.get(),
      containerizer->get(),
      flags);

  ASSERT_SOME(slave);

  MockScheduler sched;

  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      DEFAULT_CREDENTIAL);

  Future<FrameworkID> frameworkId;
  EXPECT_CALL(sched, registered(&driver, _, _))
    .WillOnce(FutureArg<1>(&frameworkId));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(frameworkId);

  AWAIT_READY(offers);
  ASSERT_FALSE(offers->empty());

  const Offer& offer = offers.get()[0];

  const string key = "iops";
  const string value = "150";

  hashmap<string, string> options = {{key, value}};

  // Create a volume with relative path.
  const string _driver = "driver";
  const string name = "name";
  const string containerPath = "tmp/foo";

  Volume volume = createDockerVolume(_driver, name, containerPath, options);
  volume.set_mode(Volume::RO);

  TaskInfo task = createTask(
      offer.slave_id(),
      offer.resources(),
      "echo 'hello' > " + containerPath + "/file");

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.add_volumes()->CopyFrom(volume);

  task.mutable_container()->CopyFrom(containerInfo);

  // Create mount point for volume.
  const string mountPoint = path::join(os::getcwd(), "volume");
  ASSERT_SOME(os::mkdir(mountPoint));

  Future<string> mountName;
  Future<hashmap<string, string>> mountOptions;

  EXPECT_CALL(*mockClient, mount(_driver, _, _))
    .WillOnce(DoAll(FutureArg<1>(&mountName),
                    FutureArg<2>(&mountOptions),
                    Return(mountPoint)));

  Future<string> unmountName;

  EXPECT_CALL(*mockClient, unmount(_driver, _))
    .WillOnce(DoAll(FutureArg<1>(&unmountName),
                    Return(Nothing())));

  Future<TaskStatus> statusStarting;
  Future<TaskStatus> statusRunning;
  Future<TaskStatus> statusFailed;

  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusStarting))
    .WillOnce(FutureArg<1>(&statusRunning))
    .WillOnce(FutureArg<1>(&statusFailed));

  driver.launchTasks(offer.id(), {task});

  AWAIT_READY(statusStarting);
  EXPECT_EQ(TASK_STARTING, statusStarting->state());

  AWAIT_READY(statusRunning);
  EXPECT_EQ(TASK_RUNNING, statusRunning->state());

  // Make sure the docker volume mount parameters are same with the
  // parameters in `containerInfo`.
  AWAIT_EXPECT_EQ(name, mountName);

  AWAIT_READY(mountOptions);
  EXPECT_SOME_EQ(value, mountOptions->get(key));

  AWAIT_READY(statusFailed);
  EXPECT_EQ(TASK_FAILED, statusFailed->state());

  // Make sure the docker volume unmount parameters are same with
  // the parameters in `containerInfo`.
  AWAIT_EXPECT_EQ(name, unmountName);

  driver.stop();
  driver.join();
}


// This test verifies that a container launched with
// a rootfs cannot write to a read-only docker volume.
TEST_F(DockerVolumeIsolatorTest,
       ROOT_INTERNET_CURL_CommandTaskRootfsWithReadOnlyVolume)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  slave::Flags flags = CreateSlaveFlags();
  flags.isolation = "docker/volume,docker/runtime,filesystem/linux";
  flags.image_providers = "docker";

  MockDockerVolumeDriverClient* mockClient = new MockDockerVolumeDriverClient;

  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer(flags, Owned<DriverClient>(mockClient));

  ASSERT_SOME(containerizer);

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(
      detector.get(),
      containerizer->get(),
      flags);

  ASSERT_SOME(slave);

  MockScheduler sched;

  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      DEFAULT_CREDENTIAL);

  Future<FrameworkID> frameworkId;
  EXPECT_CALL(sched, registered(&driver, _, _))
    .WillOnce(FutureArg<1>(&frameworkId));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(frameworkId);

  AWAIT_READY(offers);
  ASSERT_FALSE(offers->empty());

  const Offer& offer = offers.get()[0];

  const string key = "iops";
  const string value = "150";

  hashmap<string, string> options = {{key, value}};

  // Create a volume with relative path.
  const string _driver = "driver";
  const string name = "name";
  const string containerPath = "tmp/foo";

  Volume volume = createDockerVolume(_driver, name, containerPath, options);
  volume.set_mode(Volume::RO);

  TaskInfo task = createTask(
      offer.slave_id(),
      offer.resources(),
      "echo 'hello' > " + containerPath + "/file");

  Image image;
  image.set_type(Image::DOCKER);
  image.mutable_docker()->set_name("alpine");

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.add_volumes()->CopyFrom(volume);

  containerInfo.mutable_mesos()->mutable_image()->CopyFrom(image);

  task.mutable_container()->CopyFrom(containerInfo);

  // Create mount point for volume.
  const string mountPoint = path::join(os::getcwd(), "volume");
  ASSERT_SOME(os::mkdir(mountPoint));

  Future<string> mountName;
  Future<hashmap<string, string>> mountOptions;

  EXPECT_CALL(*mockClient, mount(_driver, _, _))
    .WillOnce(DoAll(FutureArg<1>(&mountName),
                    FutureArg<2>(&mountOptions),
                    Return(mountPoint)));

  Future<string> unmountName;

  EXPECT_CALL(*mockClient, unmount(_driver, _))
    .WillOnce(DoAll(FutureArg<1>(&unmountName),
                    Return(Nothing())));

  Future<TaskStatus> statusStarting;
  Future<TaskStatus> statusRunning;
  Future<TaskStatus> statusFailed;

  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusStarting))
    .WillOnce(FutureArg<1>(&statusRunning))
    .WillOnce(FutureArg<1>(&statusFailed));

  driver.launchTasks(offer.id(), {task});

  AWAIT_READY(statusStarting);
  EXPECT_EQ(TASK_STARTING, statusStarting->state());

  AWAIT_READY(statusRunning);
  EXPECT_EQ(TASK_RUNNING, statusRunning->state());

  // Make sure the docker volume mount parameters are same with the
  // parameters in `containerInfo`.
  AWAIT_EXPECT_EQ(name, mountName);

  AWAIT_READY(mountOptions);
  EXPECT_SOME_EQ(value, mountOptions->get(key));

  AWAIT_READY(statusFailed);
  EXPECT_EQ(TASK_FAILED, statusFailed->state());

  // Make sure the docker volume unmount parameters are same with
  // the parameters in `containerInfo`.
  AWAIT_EXPECT_EQ(name, unmountName);

  driver.stop();
  driver.join();
}


// This test verifies that unmount operation can be still invoked for
// a docker volume even the previous unmount operation for the same
// docker volume failed. This is a regression test for MESOS-10126.
TEST_F(DockerVolumeIsolatorTest,
       ROOT_CommandTaskNoRootfsWithUnmountVolumeFailure)
{
  Clock::pause();

  master::Flags masterFlags = CreateMasterFlags();

  Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
  ASSERT_SOME(master);

  slave::Flags flags = CreateSlaveFlags();
  flags.docker_volume_checkpoint_dir = path::join(os::getcwd(), "checkpoint");

  MockDockerVolumeDriverClient* mockClient = new MockDockerVolumeDriverClient;

  Try<Owned<MesosContainerizer>> containerizer =
    createContainerizer(flags, Owned<DriverClient>(mockClient));

  ASSERT_SOME(containerizer);

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(
      detector.get(),
      containerizer->get(),
      flags);

  ASSERT_SOME(slave);

  MockScheduler sched;

  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      DEFAULT_CREDENTIAL);

  EXPECT_CALL(sched, registered(&driver, _, _));

  Future<vector<Offer>> offers1;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers1));

  driver.start();

  Clock::advance(masterFlags.allocation_interval);

  AWAIT_READY(offers1);
  ASSERT_FALSE(offers1->empty());

  const Offer& offer1 = offers1.get()[0];

  // Create a docker volume with relative path.
  const string volumeDriver = "driver";
  const string volumeName = "name";
  const string containerPath = "tmp/foo";

  Volume volume = createDockerVolume(volumeDriver, volumeName, containerPath);

  // Launch the first task with the docker volume.
  TaskInfo task1 = createTask(
      offer1.slave_id(),
      offer1.resources(),
      "test -f " + containerPath + "/file");

  ContainerInfo containerInfo;
  containerInfo.set_type(ContainerInfo::MESOS);
  containerInfo.add_volumes()->CopyFrom(volume);

  task1.mutable_container()->CopyFrom(containerInfo);

  // Create mount point for the volume.
  const string mountPoint = path::join(os::getcwd(), "volume");
  ASSERT_SOME(os::mkdir(mountPoint));
  ASSERT_SOME(os::touch(path::join(mountPoint, "file")));

  Future<string> mountName1;
  EXPECT_CALL(*mockClient, mount(volumeDriver, _, _))
    .WillOnce(DoAll(FutureArg<1>(&mountName1),
                    Return(mountPoint)));

  // Simulate an unmount failure.
  Future<string> unmountName1;
  EXPECT_CALL(*mockClient, unmount(volumeDriver, _))
    .WillOnce(DoAll(FutureArg<1>(&unmountName1),
                    Return(Failure("Mock failure"))));

  Future<TaskStatus> statusStarting1;
  Future<TaskStatus> statusRunning1;
  Future<TaskStatus> statusFinished1;

  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusStarting1))
    .WillOnce(FutureArg<1>(&statusRunning1))
    .WillOnce(FutureArg<1>(&statusFinished1));

  Future<vector<Offer>> offers2;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers2))
    .WillRepeatedly(Return());

  driver.launchTasks(offer1.id(), {task1});

  AWAIT_READY(statusStarting1);
  EXPECT_EQ(TASK_STARTING, statusStarting1->state());

  AWAIT_READY(statusRunning1);
  EXPECT_EQ(TASK_RUNNING, statusRunning1->state());

  // Make sure the docker volume mount parameters are same with the
  // parameters in `containerInfo`.
  AWAIT_EXPECT_EQ(volumeName, mountName1);

  AWAIT_READY(statusFinished1);
  EXPECT_EQ(TASK_FINISHED, statusFinished1->state());

  Clock::resume();

  // Make sure the docker volume unmount parameters are same with
  // the parameters in `containerInfo`.
  AWAIT_EXPECT_EQ(volumeName, unmountName1);

  Clock::pause();
  Clock::settle();
  Clock::advance(masterFlags.allocation_interval);

  AWAIT_READY(offers2);
  ASSERT_FALSE(offers2->empty());

  const Offer& offer2 = offers2.get()[0];

  // Launch the second task with the same docker volume.
  TaskInfo task2 = createTask(
      offer2.slave_id(),
      offer2.resources(),
      "test -f " + containerPath + "/file");

  task2.mutable_container()->CopyFrom(containerInfo);

  Future<string> mountName2;
  EXPECT_CALL(*mockClient, mount(volumeDriver, _, _))
    .WillOnce(DoAll(FutureArg<1>(&mountName2),
                    Return(mountPoint)));

  Future<string> unmountName2;
  EXPECT_CALL(*mockClient, unmount(volumeDriver, _))
    .WillOnce(DoAll(FutureArg<1>(&unmountName2),
                    Return(Nothing())));

  Future<TaskStatus> statusStarting2;
  Future<TaskStatus> statusRunning2;
  Future<TaskStatus> statusFinished2;

  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusStarting2))
    .WillOnce(FutureArg<1>(&statusRunning2))
    .WillOnce(FutureArg<1>(&statusFinished2));

  driver.launchTasks(offer2.id(), {task2});

  AWAIT_READY(statusStarting2);
  EXPECT_EQ(TASK_STARTING, statusStarting2->state());

  AWAIT_READY(statusRunning2);
  EXPECT_EQ(TASK_RUNNING, statusRunning2->state());

  // Make sure the docker volume mount parameters are same with the
  // parameters in `containerInfo`.
  AWAIT_EXPECT_EQ(volumeName, mountName2);

  AWAIT_READY(statusFinished2);
  EXPECT_EQ(TASK_FINISHED, statusFinished2->state());

  Clock::resume();

  // Check the unmount operation can still be invoked even the
  // previous one failed.
  AWAIT_EXPECT_EQ(volumeName, unmountName2);

  driver.stop();
  driver.join();
}

} // namespace tests {
} // namespace internal {
} // namespace mesos {