| #!/usr/bin/env python |
| # coding=utf-8 |
| |
| |
| from fabric.api import env |
| from fabric.api import run |
| from fabric.api import execute |
| from fabric.api import cd |
| from fabric.api import local |
| from fabric.api import put |
| from fabric.api import sudo |
| from fabric.state import output |
| from marvin_python_toolbox import __version__ as TOOLBOX_VERSION |
| from marvin_python_toolbox.common.config import Config |
| |
| _host = Config.get("host", section="ssh_deployment").split(",") |
| _port = Config.get("port", section="ssh_deployment") |
| _user = Config.get("user", section="ssh_deployment") |
| |
| for h in _host: |
| env.hosts.append("{user}@{host}:{port}".format(user=_user, host=h, port=_port)) |
| |
| output["everything"] = False |
| output["running"] = True |
| |
| env.package = "marvin_mnist_keras_engine" |
| env.margin_engine_executor_prefix = "/opt/marvin/engine-executor" |
| env.margin_engine_executor_jar = "marvin-engine-executor-assembly-{version}.jar".format(version=TOOLBOX_VERSION) |
| env.marvin_engine_executor_path = env.margin_engine_executor_prefix + "/" + env.margin_engine_executor_jar |
| |
| |
| def install_oracle_jdk(): |
| sudo("add-apt-repository ppa:webupd8team/java -y") |
| sudo("apt-get -qq update") |
| run("echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections") |
| run("echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections") |
| sudo("apt-get install -y oracle-java8-installer") |
| |
| |
| def install_virtualenvwrapper(): |
| run("pip install virtualenvwrapper") |
| run("echo 'export WORKON_HOME=${HOME}/.virtualenvs' >> ${HOME}/.profile") |
| run("echo 'source /usr/local/bin/virtualenvwrapper.sh' >> ${HOME}/.profile") |
| |
| |
| def install_apache_spark(): |
| run("curl https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz -o /tmp/spark-2.1.1-bin-hadoop2.6.tgz") |
| sudo("tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/") |
| sudo("ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark") |
| run("echo 'export SPARK_HOME=/opt/spark' >> ${HOME}/.profile") |
| |
| |
| def install_required_packages(): |
| sudo("apt-get update -y") |
| sudo("apt-get install -y git") |
| sudo("apt-get install -y wget") |
| sudo("apt-get install -y python2.7-dev") |
| sudo("apt-get install -y python-pip") |
| sudo("apt-get install -y ipython") |
| sudo("apt-get install -y libffi-dev") |
| sudo("apt-get install -y libssl-dev") |
| sudo("apt-get install -y libxml2-dev") |
| sudo("apt-get install -y libxslt1-dev") |
| sudo("apt-get install -y libpng12-dev") |
| sudo("apt-get install -y libfreetype6-dev") |
| sudo("apt-get install -y python-tk") |
| sudo("apt-get install -y libsasl2-dev") |
| sudo("apt-get install -y python-pip") |
| sudo("apt-get install -y graphviz") |
| sudo("pip install --upgrade pip") |
| |
| |
| def install_marvin_engine_executor(): |
| sudo("mkdir -p {prefix}".format(prefix=env.margin_engine_executor_prefix)) |
| with cd("{prefix}".format(prefix=env.margin_engine_executor_prefix)): |
| sudo("wget https://s3.amazonaws.com/marvin-engine-executor/{jar}".format(jar=env.margin_engine_executor_jar)) |
| |
| |
| def create_marvin_engines_prefix(): |
| sudo("mkdir -p /opt/marvin/engines") |
| sudo("chown {user}:{user} /opt/marvin/engines".format(user=env.user)) |
| sudo("mkdir -p /var/log/marvin/engines") |
| sudo("chown {user}:{user} /var/log/marvin/engines".format(user=env.user)) |
| sudo("mkdir -p /var/run/marvin/engines") |
| sudo("chown {user}:{user} /var/run/marvin/engines".format(user=env.user)) |
| |
| |
| def configure_marvin_environment(): |
| run("echo 'export MARVIN_HOME=${HOME}/marvin' >> ${HOME}/.profile") |
| run("echo 'export MARVIN_DATA_PATH=${MARVIN_HOME}/data' >> ${HOME}/.profile") |
| run("mkdir -p ${MARVIN_HOME}") |
| run("mkdir -p ${MARVIN_DATA_PATH}") |
| |
| |
| def provision(): |
| execute(install_required_packages) |
| execute(install_virtualenvwrapper) |
| execute(install_oracle_jdk) |
| execute(install_apache_spark) |
| execute(install_marvin_engine_executor) |
| execute(create_marvin_engines_prefix) |
| execute(configure_marvin_environment) |
| |
| |
| def package(version): |
| package = env.package |
| local("mkdir -p .packages") |
| local("tar czvf .packages/{package}-{version}.tar.gz --exclude='.packages' .".format( |
| package=package, version=version)) |
| |
| |
| def deploy(version, skip_clean=False): |
| execute(engine_stop) |
| package = env.package |
| put(local_path=".packages/{package}-{version}.tar.gz".format( |
| package=package, version=version), remote_path="/tmp/") |
| run("mkdir -p /opt/marvin/engines/{package}/{version}".format( |
| package=package, version=version)) |
| with cd("/opt/marvin/engines/{package}/{version}".format( |
| package=package, version=version)): |
| run("tar xzvf /tmp/{package}-{version}.tar.gz".format( |
| package=package, version=version)) |
| with cd("/opt/marvin/engines/{package}".format(package=package)): |
| symlink_exists = run("stat current", quiet=True).succeeded |
| if (symlink_exists): |
| run("rm current") |
| run("ln -s {version} current".format(version=version)) |
| with cd("/opt/marvin/engines/{package}/current".format(package=package)): |
| run("mkvirtualenv {package}_env".format(package=package)) |
| run("setvirtualenvproject") |
| if skip_clean: |
| run("workon {package}_env && make marvin".format( |
| package=package)) |
| else: |
| run("workon {package}_env && make clean && make marvin".format( |
| package=package)) |
| execute(engine_start) |
| |
| |
| def engine_start(http_host, http_port): |
| package = env.package |
| |
| command = ( |
| "workon {package}_env &&" |
| " (marvin engine-httpserver" |
| " -h {http_host}" |
| " -p {http_port}" |
| " -e {executor}" |
| " 1> /var/log/marvin/engines/{package}.out" |
| " 2> /var/log/marvin/engines/{package}.err" |
| " & echo $! > /var/run/marvin/engines/{package}.pid)" |
| ).format( |
| package=package, |
| http_host=http_host, |
| http_port=http_port, |
| executor=env.marvin_engine_executor_path |
| ) |
| |
| with cd("/opt/marvin/engines/{package}/current".format(package=package)): |
| run(command, pty=False) |
| |
| |
| def engine_stop(): |
| package = env.package |
| |
| pid_file_exists = run("cat /var/run/marvin/engines/{package}.pid".format( |
| package=package), quiet=True) |
| if pid_file_exists.succeeded: |
| with cd("/opt/marvin/engines/{package}/current".format(package=package)): |
| children_pids = run("ps --ppid $(cat /var/run/marvin/engines/{package}.pid) -o pid --no-headers |xargs echo".format( |
| package=package)) |
| run("kill $(cat /var/run/marvin/engines/{package}.pid) {children_pids}".format( |
| package=package, children_pids=children_pids)) |
| run("rm /var/run/marvin/engines/{package}.pid".format(package=package)) |
| |
| |
| def engine_status(): |
| package = env.package |
| pid_file_exists = run("cat /var/run/marvin/engines/{package}.pid".format( |
| package=package), quiet=True) |
| if pid_file_exists.succeeded: |
| is_running = run("ps $(cat /var/run/marvin/engines/{package}.pid)".format(package=package), quiet=True) |
| if is_running.succeeded: |
| print "Your engine is running :)" |
| else: |
| print "Your engine is not running :(" |
| else: |
| print "Your engine is not running :(" |