buildstream/sandbox/_sandboxbwrap.py - buildstream - Git at Google

 #
 #  Copyright (C) 2016 Codethink Limited
 #
 #  This program is free software; you can redistribute it and/or
 #  modify it under the terms of the GNU Lesser General Public
 #  License as published by the Free Software Foundation; either
 #  version 2 of the License, or (at your option) any later version.
 #
 #  This library is distributed in the hope that it will be useful,
 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
 #  Lesser General Public License for more details.
 #
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 #
 #  Authors:
 #        Andrew Leeming <andrew.leeming@codethink.co.uk>
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 import collections
 import json
 import os
 import sys
 import time
 import errno
 import signal
 import subprocess
 import shutil
 from contextlib import ExitStack, suppress
 from tempfile import TemporaryFile

 import psutil

 from .._exceptions import SandboxError
 from .. import utils, _signals
 from ._mount import MountMap
 from . import Sandbox, SandboxFlags


 # SandboxBwrap()
 #
 # Default bubblewrap based sandbox implementation.
 #
 class SandboxBwrap(Sandbox):

     # Minimal set of devices for the sandbox
     DEVICES = [
         '/dev/full',
         '/dev/null',
         '/dev/urandom',
         '/dev/random',
         '/dev/zero'
     ]

     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.user_ns_available = kwargs['user_ns_available']
         self.die_with_parent_available = kwargs['die_with_parent_available']
         self.json_status_available = kwargs['json_status_available']
         self.linux32 = kwargs['linux32']

     def _run(self, command, flags, *, cwd, env):
         stdout, stderr = self._get_output()

         # Allowable access to underlying storage as we're part of the sandbox
         root_directory = self.get_virtual_directory()._get_underlying_directory()

         if not self._has_command(command[0], env):
             raise SandboxError("Staged artifacts do not provide command "
                                "'{}'".format(command[0]),
                                reason='missing-command')

         # Create the mount map, this will tell us where
         # each mount point needs to be mounted from and to
         mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY)
         root_mount_source = mount_map.get_mount_source('/')

         # start command with linux32 if needed
         if self.linux32:
             bwrap_command = [utils.get_host_tool('linux32')]
         else:
             bwrap_command = []

         # Grab the full path of the bwrap binary
         bwrap_command += [utils.get_host_tool('bwrap')]

         for k, v in env.items():
             bwrap_command += ['--setenv', k, v]
         for k in os.environ.keys() - env.keys():
             bwrap_command += ['--unsetenv', k]

         # Create a new pid namespace, this also ensures that any subprocesses
         # are cleaned up when the bwrap process exits.
         bwrap_command += ['--unshare-pid']

         # Ensure subprocesses are cleaned up when the bwrap parent dies.
         if self.die_with_parent_available:
             bwrap_command += ['--die-with-parent']

         # Add in the root filesystem stuff first.
         #
         # The rootfs is mounted as RW initially so that further mounts can be
         # placed on top. If a RO root is required, after all other mounts are
         # complete, root is remounted as RO
         bwrap_command += ["--bind", root_mount_source, "/"]

         if not flags & SandboxFlags.NETWORK_ENABLED:
             bwrap_command += ['--unshare-net']
             bwrap_command += ['--unshare-uts', '--hostname', 'buildstream']
             bwrap_command += ['--unshare-ipc']

         # Give it a proc and tmpfs
         bwrap_command += [
             '--proc', '/proc',
             '--tmpfs', '/tmp'
         ]

         # In interactive mode, we want a complete devpts inside
         # the container, so there is a /dev/console and such. In
         # the regular non-interactive sandbox, we want to hand pick
         # a minimal set of devices to expose to the sandbox.
         #
         if flags & SandboxFlags.INTERACTIVE:
             bwrap_command += ['--dev', '/dev']
         else:
             for device in self.DEVICES:
                 bwrap_command += ['--dev-bind', device, device]

         # Add bind mounts to any marked directories
         marked_directories = self._get_marked_directories()
         mount_source_overrides = self._get_mount_sources()
         for mark in marked_directories:
             mount_point = mark['directory']
             if mount_point in mount_source_overrides:  # pylint: disable=consider-using-get
                 mount_source = mount_source_overrides[mount_point]
             else:
                 mount_source = mount_map.get_mount_source(mount_point)

             # Use --dev-bind for all mounts, this is simply a bind mount which does
             # not restrictive about devices.
             #
             # While it's important for users to be able to mount devices
             # into the sandbox for `bst shell` testing purposes, it is
             # harmless to do in a build environment where the directories
             # we mount just never contain device files.
             #
             bwrap_command += ['--dev-bind', mount_source, mount_point]

         if flags & SandboxFlags.ROOT_READ_ONLY:
             bwrap_command += ["--remount-ro", "/"]

         if cwd is not None:
             bwrap_command += ['--dir', cwd]
             bwrap_command += ['--chdir', cwd]

         # Set UID and GUI
         if self.user_ns_available:
             bwrap_command += ['--unshare-user']
             if not flags & SandboxFlags.INHERIT_UID:
                 uid = self._get_config().build_uid
                 gid = self._get_config().build_gid
                 bwrap_command += ['--uid', str(uid), '--gid', str(gid)]

         with ExitStack() as stack:
             pass_fds = ()
             # Improve error reporting with json-status if available
             if self.json_status_available:
                 json_status_file = stack.enter_context(TemporaryFile())
                 pass_fds = (json_status_file.fileno(),)
                 bwrap_command += ['--json-status-fd', str(json_status_file.fileno())]

             # Add the command
             bwrap_command += command

             # bwrap might create some directories while being suid
             # and may give them to root gid, if it does, we'll want
             # to clean them up after, so record what we already had
             # there just in case so that we can safely cleanup the debris.
             #
             existing_basedirs = {
                 directory: os.path.exists(os.path.join(root_directory, directory))
                 for directory in ['tmp', 'dev', 'proc']
             }

             # Use the MountMap context manager to ensure that any redirected
             # mounts through fuse layers are in context and ready for bwrap
             # to mount them from.
             #
             stack.enter_context(mount_map.mounted(self))

             # If we're interactive, we want to inherit our stdin,
             # otherwise redirect to /dev/null, ensuring process
             # disconnected from terminal.
             if flags & SandboxFlags.INTERACTIVE:
                 stdin = sys.stdin
             else:
                 stdin = stack.enter_context(open(os.devnull, "r"))

             # Run bubblewrap !
             exit_code = self.run_bwrap(bwrap_command, stdin, stdout, stderr,
                                        (flags & SandboxFlags.INTERACTIVE), pass_fds)

             # Cleanup things which bwrap might have left behind, while
             # everything is still mounted because bwrap can be creating
             # the devices on the fuse mount, so we should remove it there.
             if not flags & SandboxFlags.INTERACTIVE:
                 for device in self.DEVICES:
                     device_path = os.path.join(root_mount_source, device.lstrip('/'))

                     # This will remove the device in a loop, allowing some
                     # retries in case the device file leaked by bubblewrap is still busy
                     self.try_remove_device(device_path)

             # Remove /tmp, this is a bwrap owned thing we want to be sure
             # never ends up in an artifact
             for basedir in ['tmp', 'dev', 'proc']:

                 # Skip removal of directories which already existed before
                 # launching bwrap
                 if existing_basedirs[basedir]:
                     continue

                 base_directory = os.path.join(root_mount_source, basedir)

                 if flags & SandboxFlags.INTERACTIVE:
                     # Be more lenient in interactive mode here.
                     #
                     # In interactive mode; it's possible that the project shell
                     # configuration has mounted some things below the base
                     # directories, such as /dev/dri, and in this case it's less
                     # important to consider cleanup, as we wont be collecting
                     # this build result and creating an artifact.
                     #
                     # Note: Ideally; we should instead fix upstream bubblewrap to
                     #       cleanup any debris it creates at startup time, and do
                     #       the same ourselves for any directories we explicitly create.
                     #
                     shutil.rmtree(base_directory, ignore_errors=True)
                 else:
                     try:
                         os.rmdir(base_directory)
                     except FileNotFoundError:
                         # ignore this, if bwrap cleaned up properly then it's not a problem.
                         #
                         # If the directory was not empty on the other hand, then this is clearly
                         # a bug, bwrap mounted a tempfs here and when it exits, that better be empty.
                         pass

             if self.json_status_available:
                 json_status_file.seek(0, 0)
                 child_exit_code = None
                 # The JSON status file's output is a JSON object per line
                 # with the keys present identifying the type of message.
                 # The only message relevant to us now is the exit-code of the subprocess.
                 for line in json_status_file:
                     with suppress(json.decoder.JSONDecodeError):
                         o = json.loads(line)
                         if isinstance(o, collections.abc.Mapping) and 'exit-code' in o:
                             child_exit_code = o['exit-code']
                             break
                 if child_exit_code is None:
                     raise SandboxError("`bwrap' terminated during sandbox setup with exitcode {}".format(exit_code),
                                        reason="bwrap-sandbox-fail")
                 exit_code = child_exit_code

         self._vdir._mark_changed()
         return exit_code

     def run_bwrap(self, argv, stdin, stdout, stderr, interactive, pass_fds):
         # Wrapper around subprocess.Popen() with common settings.
         #
         # This function blocks until the subprocess has terminated.
         #
         # It then returns a tuple of (exit code, stdout output, stderr output).
         # If stdout was not equal to subprocess.PIPE, stdout will be None. Same for
         # stderr.

         # Fetch the process actually launched inside the bwrap sandbox, or the
         # intermediat control bwrap processes.
         #
         # NOTE:
         #   The main bwrap process itself is setuid root and as such we cannot
         #   send it any signals. Since we launch bwrap with --unshare-pid, it's
         #   direct child is another bwrap process which retains ownership of the
         #   pid namespace. This is the right process to kill when terminating.
         #
         #   The grandchild is the binary which we asked bwrap to launch on our
         #   behalf, whatever this binary is, it is the right process to use
         #   for suspending and resuming. In the case that this is a shell, the
         #   shell will be group leader and all build scripts will stop/resume
         #   with that shell.
         #
         def get_user_proc(bwrap_pid, grand_child=False):
             bwrap_proc = psutil.Process(bwrap_pid)
             bwrap_children = bwrap_proc.children()
             if bwrap_children:
                 if grand_child:
                     bwrap_grand_children = bwrap_children[0].children()
                     if bwrap_grand_children:
                         return bwrap_grand_children[0]
                 else:
                     return bwrap_children[0]
             return None

         def terminate_bwrap():
             if process:
                 user_proc = get_user_proc(process.pid)
                 if user_proc:
                     user_proc.kill()

         def suspend_bwrap():
             if process:
                 user_proc = get_user_proc(process.pid, grand_child=True)
                 if user_proc:
                     group_id = os.getpgid(user_proc.pid)
                     os.killpg(group_id, signal.SIGSTOP)

         def resume_bwrap():
             if process:
                 user_proc = get_user_proc(process.pid, grand_child=True)
                 if user_proc:
                     group_id = os.getpgid(user_proc.pid)
                     os.killpg(group_id, signal.SIGCONT)

         with ExitStack() as stack:

             # We want to launch bwrap in a new session in non-interactive
             # mode so that we handle the SIGTERM and SIGTSTP signals separately
             # from the nested bwrap process, but in interactive mode this
             # causes launched shells to lack job control (we dont really
             # know why that is).
             #
             if interactive:
                 new_session = False
             else:
                 new_session = True
                 stack.enter_context(_signals.suspendable(suspend_bwrap, resume_bwrap))
                 stack.enter_context(_signals.terminator(terminate_bwrap))

             process = subprocess.Popen(
                 argv,
                 # The default is to share file descriptors from the parent process
                 # to the subprocess, which is rarely good for sandboxing.
                 close_fds=True,
                 pass_fds=pass_fds,
                 stdin=stdin,
                 stdout=stdout,
                 stderr=stderr,
                 start_new_session=new_session
             )

             # Wait for the child process to finish, ensuring that
             # a SIGINT has exactly the effect the user probably
             # expects (i.e. let the child process handle it).
             try:
                 while True:
                     try:
                         _, status = os.waitpid(process.pid, 0)
                         # If the process exits due to a signal, we
                         # brutally murder it to avoid zombies
                         if not os.WIFEXITED(status):
                             user_proc = get_user_proc(process.pid)
                             if user_proc:
                                 utils._kill_process_tree(user_proc.pid)

                     # If we receive a KeyboardInterrupt we continue
                     # waiting for the process since we are in the same
                     # process group and it should also have received
                     # the SIGINT.
                     except KeyboardInterrupt:
                         continue

                     break
             # If we can't find the process, it has already died of its
             # own accord, and therefore we don't need to check or kill
             # anything.
             except psutil.NoSuchProcess:
                 pass

             # Return the exit code - see the documentation for
             # os.WEXITSTATUS to see why this is required.
             if os.WIFEXITED(status):
                 exit_code = os.WEXITSTATUS(status)
             else:
                 exit_code = -1

             if interactive and stdin.isatty():
                 # Make this process the foreground process again, otherwise the
                 # next read() on stdin will trigger SIGTTIN and stop the process.
                 # This is required because the sandboxed process does not have
                 # permission to do this on its own (running in separate PID namespace).
                 #
                 # tcsetpgrp() will trigger SIGTTOU when called from a background
                 # process, so ignore it temporarily.
                 handler = signal.signal(signal.SIGTTOU, signal.SIG_IGN)
                 os.tcsetpgrp(0, os.getpid())
                 signal.signal(signal.SIGTTOU, handler)

         return exit_code

     def try_remove_device(self, device_path):

         # Put some upper limit on the tries here
         max_tries = 1000
         tries = 0

         while True:
             try:
                 os.unlink(device_path)
             except OSError as e:
                 if e.errno == errno.EBUSY:
                     # This happens on some machines, seems there is a race sometimes
                     # after bubblewrap returns and the device files it bind-mounted did
                     # not finish unmounting.
                     #
                     if tries < max_tries:
                         tries += 1
                         time.sleep(1 / 100)
                         continue
                     else:
                         # We've reached the upper limit of tries, bail out now
                         # because something must have went wrong
                         #
                         raise
                 elif e.errno == errno.ENOENT:
                     # Bubblewrap cleaned it up for us, no problem if we cant remove it
                     break
                 else:
                     # Something unexpected, reraise this error
                     raise
             else:
                 # Successfully removed the symlink
                 break
	#
	# Copyright (C) 2016 Codethink Limited
	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU Lesser General Public
	# License as published by the Free Software Foundation; either
	# version 2 of the License, or (at your option) any later version.
	#
	# This library is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	# Lesser General Public License for more details.
	#
	# You should have received a copy of the GNU Lesser General Public
	# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	#
	# Authors:
	# Andrew Leeming <andrew.leeming@codethink.co.uk>
	# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
	import collections
	import json
	import os
	import sys
	import time
	import errno
	import signal
	import subprocess
	import shutil
	from contextlib import ExitStack, suppress
	from tempfile import TemporaryFile

	import psutil

	from .._exceptions import SandboxError
	from .. import utils, _signals
	from ._mount import MountMap
	from . import Sandbox, SandboxFlags


	# SandboxBwrap()
	#
	# Default bubblewrap based sandbox implementation.
	#
	class SandboxBwrap(Sandbox):

	# Minimal set of devices for the sandbox
	DEVICES = [
	'/dev/full',
	'/dev/null',
	'/dev/urandom',
	'/dev/random',
	'/dev/zero'
	]

	def __init__(self, args, *kwargs):
	super().__init__(args, *kwargs)
	self.user_ns_available = kwargs['user_ns_available']
	self.die_with_parent_available = kwargs['die_with_parent_available']
	self.json_status_available = kwargs['json_status_available']
	self.linux32 = kwargs['linux32']

	def _run(self, command, flags, *, cwd, env):
	stdout, stderr = self._get_output()

	# Allowable access to underlying storage as we're part of the sandbox
	root_directory = self.get_virtual_directory()._get_underlying_directory()

	if not self._has_command(command[0], env):
	raise SandboxError("Staged artifacts do not provide command "
	"'{}'".format(command[0]),
	reason='missing-command')

	# Create the mount map, this will tell us where
	# each mount point needs to be mounted from and to
	mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY)
	root_mount_source = mount_map.get_mount_source('/')

	# start command with linux32 if needed
	if self.linux32:
	bwrap_command = [utils.get_host_tool('linux32')]
	else:
	bwrap_command = []

	# Grab the full path of the bwrap binary
	bwrap_command += [utils.get_host_tool('bwrap')]

	for k, v in env.items():
	bwrap_command += ['--setenv', k, v]
	for k in os.environ.keys() - env.keys():
	bwrap_command += ['--unsetenv', k]

	# Create a new pid namespace, this also ensures that any subprocesses
	# are cleaned up when the bwrap process exits.
	bwrap_command += ['--unshare-pid']

	# Ensure subprocesses are cleaned up when the bwrap parent dies.
	if self.die_with_parent_available:
	bwrap_command += ['--die-with-parent']

	# Add in the root filesystem stuff first.
	#
	# The rootfs is mounted as RW initially so that further mounts can be
	# placed on top. If a RO root is required, after all other mounts are
	# complete, root is remounted as RO
	bwrap_command += ["--bind", root_mount_source, "/"]

	if not flags & SandboxFlags.NETWORK_ENABLED:
	bwrap_command += ['--unshare-net']
	bwrap_command += ['--unshare-uts', '--hostname', 'buildstream']
	bwrap_command += ['--unshare-ipc']

	# Give it a proc and tmpfs
	bwrap_command += [
	'--proc', '/proc',
	'--tmpfs', '/tmp'
	]

	# In interactive mode, we want a complete devpts inside
	# the container, so there is a /dev/console and such. In
	# the regular non-interactive sandbox, we want to hand pick
	# a minimal set of devices to expose to the sandbox.
	#
	if flags & SandboxFlags.INTERACTIVE:
	bwrap_command += ['--dev', '/dev']
	else:
	for device in self.DEVICES:
	bwrap_command += ['--dev-bind', device, device]

	# Add bind mounts to any marked directories
	marked_directories = self._get_marked_directories()
	mount_source_overrides = self._get_mount_sources()
	for mark in marked_directories:
	mount_point = mark['directory']
	if mount_point in mount_source_overrides: # pylint: disable=consider-using-get
	mount_source = mount_source_overrides[mount_point]
	else:
	mount_source = mount_map.get_mount_source(mount_point)

	# Use --dev-bind for all mounts, this is simply a bind mount which does
	# not restrictive about devices.
	#
	# While it's important for users to be able to mount devices
	# into the sandbox for `bst shell` testing purposes, it is
	# harmless to do in a build environment where the directories
	# we mount just never contain device files.
	#
	bwrap_command += ['--dev-bind', mount_source, mount_point]

	if flags & SandboxFlags.ROOT_READ_ONLY:
	bwrap_command += ["--remount-ro", "/"]

	if cwd is not None:
	bwrap_command += ['--dir', cwd]
	bwrap_command += ['--chdir', cwd]

	# Set UID and GUI
	if self.user_ns_available:
	bwrap_command += ['--unshare-user']
	if not flags & SandboxFlags.INHERIT_UID:
	uid = self._get_config().build_uid
	gid = self._get_config().build_gid
	bwrap_command += ['--uid', str(uid), '--gid', str(gid)]

	with ExitStack() as stack:
	pass_fds = ()
	# Improve error reporting with json-status if available
	if self.json_status_available:
	json_status_file = stack.enter_context(TemporaryFile())
	pass_fds = (json_status_file.fileno(),)
	bwrap_command += ['--json-status-fd', str(json_status_file.fileno())]

	# Add the command
	bwrap_command += command

	# bwrap might create some directories while being suid
	# and may give them to root gid, if it does, we'll want
	# to clean them up after, so record what we already had
	# there just in case so that we can safely cleanup the debris.
	#
	existing_basedirs = {
	directory: os.path.exists(os.path.join(root_directory, directory))
	for directory in ['tmp', 'dev', 'proc']
	}

	# Use the MountMap context manager to ensure that any redirected
	# mounts through fuse layers are in context and ready for bwrap
	# to mount them from.
	#
	stack.enter_context(mount_map.mounted(self))

	# If we're interactive, we want to inherit our stdin,
	# otherwise redirect to /dev/null, ensuring process
	# disconnected from terminal.
	if flags & SandboxFlags.INTERACTIVE:
	stdin = sys.stdin
	else:
	stdin = stack.enter_context(open(os.devnull, "r"))

	# Run bubblewrap !
	exit_code = self.run_bwrap(bwrap_command, stdin, stdout, stderr,
	(flags & SandboxFlags.INTERACTIVE), pass_fds)

	# Cleanup things which bwrap might have left behind, while
	# everything is still mounted because bwrap can be creating
	# the devices on the fuse mount, so we should remove it there.
	if not flags & SandboxFlags.INTERACTIVE:
	for device in self.DEVICES:
	device_path = os.path.join(root_mount_source, device.lstrip('/'))

	# This will remove the device in a loop, allowing some
	# retries in case the device file leaked by bubblewrap is still busy
	self.try_remove_device(device_path)

	# Remove /tmp, this is a bwrap owned thing we want to be sure
	# never ends up in an artifact
	for basedir in ['tmp', 'dev', 'proc']:

	# Skip removal of directories which already existed before
	# launching bwrap
	if existing_basedirs[basedir]:
	continue

	base_directory = os.path.join(root_mount_source, basedir)

	if flags & SandboxFlags.INTERACTIVE:
	# Be more lenient in interactive mode here.
	#
	# In interactive mode; it's possible that the project shell
	# configuration has mounted some things below the base
	# directories, such as /dev/dri, and in this case it's less
	# important to consider cleanup, as we wont be collecting
	# this build result and creating an artifact.
	#
	# Note: Ideally; we should instead fix upstream bubblewrap to
	# cleanup any debris it creates at startup time, and do
	# the same ourselves for any directories we explicitly create.
	#
	shutil.rmtree(base_directory, ignore_errors=True)
	else:
	try:
	os.rmdir(base_directory)
	except FileNotFoundError:
	# ignore this, if bwrap cleaned up properly then it's not a problem.
	#
	# If the directory was not empty on the other hand, then this is clearly
	# a bug, bwrap mounted a tempfs here and when it exits, that better be empty.
	pass

	if self.json_status_available:
	json_status_file.seek(0, 0)
	child_exit_code = None
	# The JSON status file's output is a JSON object per line
	# with the keys present identifying the type of message.
	# The only message relevant to us now is the exit-code of the subprocess.
	for line in json_status_file:
	with suppress(json.decoder.JSONDecodeError):
	o = json.loads(line)
	if isinstance(o, collections.abc.Mapping) and 'exit-code' in o:
	child_exit_code = o['exit-code']
	break
	if child_exit_code is None:
	raise SandboxError("`bwrap' terminated during sandbox setup with exitcode {}".format(exit_code),
	reason="bwrap-sandbox-fail")
	exit_code = child_exit_code

	self._vdir._mark_changed()
	return exit_code

	def run_bwrap(self, argv, stdin, stdout, stderr, interactive, pass_fds):
	# Wrapper around subprocess.Popen() with common settings.
	#
	# This function blocks until the subprocess has terminated.
	#
	# It then returns a tuple of (exit code, stdout output, stderr output).
	# If stdout was not equal to subprocess.PIPE, stdout will be None. Same for
	# stderr.

	# Fetch the process actually launched inside the bwrap sandbox, or the
	# intermediat control bwrap processes.
	#
	# NOTE:
	# The main bwrap process itself is setuid root and as such we cannot
	# send it any signals. Since we launch bwrap with --unshare-pid, it's
	# direct child is another bwrap process which retains ownership of the
	# pid namespace. This is the right process to kill when terminating.
	#
	# The grandchild is the binary which we asked bwrap to launch on our
	# behalf, whatever this binary is, it is the right process to use
	# for suspending and resuming. In the case that this is a shell, the
	# shell will be group leader and all build scripts will stop/resume
	# with that shell.
	#
	def get_user_proc(bwrap_pid, grand_child=False):
	bwrap_proc = psutil.Process(bwrap_pid)
	bwrap_children = bwrap_proc.children()
	if bwrap_children:
	if grand_child:
	bwrap_grand_children = bwrap_children[0].children()
	if bwrap_grand_children:
	return bwrap_grand_children[0]
	else:
	return bwrap_children[0]
	return None

	def terminate_bwrap():
	if process:
	user_proc = get_user_proc(process.pid)
	if user_proc:
	user_proc.kill()

	def suspend_bwrap():
	if process:
	user_proc = get_user_proc(process.pid, grand_child=True)
	if user_proc:
	group_id = os.getpgid(user_proc.pid)
	os.killpg(group_id, signal.SIGSTOP)

	def resume_bwrap():
	if process:
	user_proc = get_user_proc(process.pid, grand_child=True)
	if user_proc:
	group_id = os.getpgid(user_proc.pid)
	os.killpg(group_id, signal.SIGCONT)

	with ExitStack() as stack:

	# We want to launch bwrap in a new session in non-interactive
	# mode so that we handle the SIGTERM and SIGTSTP signals separately
	# from the nested bwrap process, but in interactive mode this
	# causes launched shells to lack job control (we dont really
	# know why that is).
	#
	if interactive:
	new_session = False
	else:
	new_session = True
	stack.enter_context(_signals.suspendable(suspend_bwrap, resume_bwrap))
	stack.enter_context(_signals.terminator(terminate_bwrap))

	process = subprocess.Popen(
	argv,
	# The default is to share file descriptors from the parent process
	# to the subprocess, which is rarely good for sandboxing.
	close_fds=True,
	pass_fds=pass_fds,
	stdin=stdin,
	stdout=stdout,
	stderr=stderr,
	start_new_session=new_session
	)

	# Wait for the child process to finish, ensuring that
	# a SIGINT has exactly the effect the user probably
	# expects (i.e. let the child process handle it).
	try:
	while True:
	try:
	_, status = os.waitpid(process.pid, 0)
	# If the process exits due to a signal, we
	# brutally murder it to avoid zombies
	if not os.WIFEXITED(status):
	user_proc = get_user_proc(process.pid)
	if user_proc:
	utils._kill_process_tree(user_proc.pid)

	# If we receive a KeyboardInterrupt we continue
	# waiting for the process since we are in the same
	# process group and it should also have received
	# the SIGINT.
	except KeyboardInterrupt:
	continue

	break
	# If we can't find the process, it has already died of its
	# own accord, and therefore we don't need to check or kill
	# anything.
	except psutil.NoSuchProcess:
	pass

	# Return the exit code - see the documentation for
	# os.WEXITSTATUS to see why this is required.
	if os.WIFEXITED(status):
	exit_code = os.WEXITSTATUS(status)
	else:
	exit_code = -1

	if interactive and stdin.isatty():
	# Make this process the foreground process again, otherwise the
	# next read() on stdin will trigger SIGTTIN and stop the process.
	# This is required because the sandboxed process does not have
	# permission to do this on its own (running in separate PID namespace).
	#
	# tcsetpgrp() will trigger SIGTTOU when called from a background
	# process, so ignore it temporarily.
	handler = signal.signal(signal.SIGTTOU, signal.SIG_IGN)
	os.tcsetpgrp(0, os.getpid())
	signal.signal(signal.SIGTTOU, handler)

	return exit_code

	def try_remove_device(self, device_path):

	# Put some upper limit on the tries here
	max_tries = 1000
	tries = 0

	while True:
	try:
	os.unlink(device_path)
	except OSError as e:
	if e.errno == errno.EBUSY:
	# This happens on some machines, seems there is a race sometimes
	# after bubblewrap returns and the device files it bind-mounted did
	# not finish unmounting.
	#
	if tries < max_tries:
	tries += 1
	time.sleep(1 / 100)
	continue
	else:
	# We've reached the upper limit of tries, bail out now
	# because something must have went wrong
	#
	raise
	elif e.errno == errno.ENOENT:
	# Bubblewrap cleaned it up for us, no problem if we cant remove it
	break
	else:
	# Something unexpected, reraise this error
	raise
	else:
	# Successfully removed the symlink
	break