Merge pull request #1 from apache/develop
Pull
diff --git a/.travis.yml b/.travis.yml
index 63f9acc..a28e24e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,4 @@
-# Copyright [2018] Apache Marvin-AI
+# Copyright [2018] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@
- language: scala
scala:
- 2.12.6
- jdk: oraclejdk8
+ jdk: openjdk8
cache:
directories:
- $HOME/.sbt
@@ -31,6 +31,8 @@
- cd engine-executor
- unset SBT_OPTS
script: travis_retry sbt -batch ++$TRAVIS_SCALA_VERSION coverage test coverageReport
+ after_success:
+ - bash <(curl -s https://codecov.io/bash)
# Python Toolbox
# Python Toolbox - Linux
- language: python
@@ -47,10 +49,11 @@
- export MARVIN_HOME=./marvin_home
- export MARVIN_DATA_PATH=./marvin_data
- export SPARK_HOME=./spark-2.1.1-bin-hadoop2.6
+ - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update ; fi
+ - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew unlink python@2 ; fi
+ - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install openssl graphviz ; fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py ; fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo python get-pip.py ; fi
- - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update ; fi
- - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install openssl graphviz ; fi
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libsasl2-dev python-pip graphviz -y ; fi
- travis_retry sudo pip install --upgrade pip
- travis_retry sudo pip install virtualenvwrapper --ignore-installed six
@@ -75,10 +78,11 @@
- export MARVIN_HOME=./marvin_home
- export MARVIN_DATA_PATH=./marvin_data
- export SPARK_HOME=./spark-2.1.1-bin-hadoop2.6
+ - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update ; fi
+ - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew unlink python@2 ; fi
+ - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install openssl graphviz ; fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py ; fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo python get-pip.py ; fi
- - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update ; fi
- - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install openssl graphviz ; fi
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libsasl2-dev python-pip graphviz -y ; fi
- travis_retry sudo pip install --upgrade pip
- travis_retry sudo pip install virtualenvwrapper --ignore-installed six
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..f0a548c
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,49 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+.PHONY: help docker-base docker-python docker-r toolbox engine-server
+
+help:
+ @echo " docker-base"
+ @echo " Builds the base docker image with common dependecies among languages."
+ @echo " docker-python"
+ @echo " Builds a docker image that can run engines in Python."
+ @echo " docker-r"
+ @echo " Builds a docker image that can run engines in R."
+ @echo " toolbox"
+ @echo " Builds the toolbox and make it available for be included into docker images."
+ @echo " engine-server"
+ @echo " Builds a jar with the engine interpreter server."
+
+docker-base:
+ docker build -t marvin-base -f engine-server/build/Dockerfile engine-server/build
+
+docker-python:
+ $(MAKE) engine-server
+ $(MAKE) toolbox
+ docker build -t marvin-python -f engine-server/build/docker-python/Dockerfile engine-server/build
+
+docker-r:
+ $(MAKE) engine-server
+ $(MAKE) toolbox
+ docker build -t marvin-r -f engine-server/build/docker-r/Dockerfile engine-server/build
+
+engine-server:
+ cd engine-server && $(MAKE) package
+ mv engine-server/target/scala-2.12/marvin-engine-server-assembly-*.jar engine-server/build/marvin-engine-server-assembly.jar
+
+toolbox:
+ tar -cf engine-server/build/python-toolbox.tgz python-toolbox
\ No newline at end of file
diff --git a/README.md b/README.md
index cf54949..f890222 100644
--- a/README.md
+++ b/README.md
@@ -15,13 +15,13 @@
limitations under the License.
-->
-# [Apache Marvin AI Platform](https://marvin.apache.org/)
+# [Apache Marvin AI Platform](https://marvin.apache.org)
Branch | Status | CodeCov
-|-|-
-[master](https://github.com/apache/incubator-marvin) | [](https://travis-ci.org/apache/incubator-marvin) | [](https://codecov.io/gh/apache/incubator-marvin)
+[master](https://github.com/apache/incubator-marvin/tree/master) | [](https://travis-ci.org/apache/incubator-marvin) | [](https://codecov.io/gh/apache/incubator-marvin)
[develop](https://github.com/apache/incubator-marvin/tree/develop) | [](https://travis-ci.org/apache/incubator-marvin/branches) | [](https://codecov.io/gh/apache/incubator-marvin/branch/develop)
**Marvin** is an open-source Artificial Intelligence platform that focuses on helping data scientists deliver meaningful solutions to complex problems. Supported by a standardized large-scale, language-agnostic architecture, Marvin simplifies the process of exploration and modeling.
diff --git a/build/Dockerfile b/build/Dockerfile
new file mode 100644
index 0000000..bbf5394
--- /dev/null
+++ b/build/Dockerfile
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##############################################################
+# Base docker image for Marvin
+##############################################################
+FROM ubuntu:16.04
+
+ENV SLEEP_MILLIS 0
+
+USER root
+
+##############################################################
+# Creates the marvin user
+##############################################################
+RUN groupadd marvin && \
+ useradd marvin -mg marvin
+
+##############################################################
+# Define all environment variables to be used
+##############################################################
+
+ENV MARVIN_HOME=/home/marvin
+ENV MARVIN_DATA_PATH=/home/marvin/marvin-data
+ENV MARVIN_ENGINE_HOME=$MARVIN_HOME/engine
+ENV MARVIN_ENGINE_ENV=marvin-engine-env
+
+
+##############################################################
+# Create all folders needed
+##############################################################
+
+RUN mkdir -p $MARVIN_HOME && \
+ mkdir -p $MARVIN_DATA_PATH && \
+ mkdir -p $MARVIN_ENGINE_HOME && \
+ mkdir -p /var/log/marvin/engines && \
+ mkdir -p /var/run/marvin/engines
+
+
+##############################################################
+# Install the system dependencies for default installation
+##############################################################
+
+RUN apt-get update -y && \
+ apt-get install -y build-essential && \
+ apt-get install -y maven git cmake software-properties-common curl libstdc++6 && \
+ apt-get install -y wget && \
+ apt-get install -y libffi-dev && \
+ apt-get install -y libssl-dev && \
+ apt-get install -y libxml2-dev && \
+ apt-get install -y libxslt1-dev && \
+ apt-get install -y libpng12-dev && \
+ apt-get install -y libfreetype6-dev && \
+ apt-get install -y libsasl2-dev && \
+ apt-get install -y graphviz && \
+ apt-get clean
+
+### Installs Open JDK 8
+RUN add-apt-repository ppa:openjdk-r/ppa && \
+ apt-get update && \
+ apt-get install -y openjdk-8-jdk
+
+## TODO - Think in a good way to make Spark an option as soon we implement docker supporting it
+##############################################################
+# Install Apache Spark
+#
+# Uncomment if you are using spark, note that is needed the
+# spark configuration files to the think works correctly.
+##############################################################
+
+# RUN curl https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz -o /tmp/spark-2.1.1-bin-hadoop2.6.tgz && \
+# tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/ && \
+# ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark
+
+# Add the b2w datalake config for Spark
+# ADD spark-conf.tar $SPARK_CONF_DIR
+# RUN mkdir -p $SPARK_CONF_DIR
\ No newline at end of file
diff --git a/build/docker-python/Dockerfile b/build/docker-python/Dockerfile
new file mode 100644
index 0000000..695be4c
--- /dev/null
+++ b/build/docker-python/Dockerfile
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_TAG=latest
+FROM marvin-base:${BASE_TAG}
+
+##############################################################
+# Define all environment variables to be used
+##############################################################
+ENV WORKON_HOME=$MARVIN_HOME/.virtualenvs
+
+##############################################################
+# Install python dependencies
+##############################################################
+
+RUN apt-get update -y && \
+ apt-get install -y python && \
+ apt-get install -y python2.7-dev && \
+ apt-get install -y python-pip && \
+ apt-get install -y ipython && \
+ # ??? apt-get install -y python-tk && \ ??? #
+ pip install --upgrade pip==9.0.1 && \
+ apt install -y libzmq3-dev libcurl4-openssl-dev libssl-dev && \
+ pip install jupyter && \
+ apt-get clean
+
+##############################################################
+# Copy and Install the marvin engine inside virtualenv
+##############################################################
+
+# adds the package containing the user-generated engine
+ADD python-toolbox.tgz $MARVIN_ENGINE_HOME
+
+# adds the freshly built engine server jar
+ADD marvin-engine-executor-assembly.jar $MARVIN_DATA_PATH
+
+##############################################################
+# Starts the jupyter http server
+##############################################################
+
+EXPOSE 8888
+
+USER marvin
+
+RUN /bin/bash -c "cd $MARVIN_ENGINE_HOME/python-toolbox && pip install . --user"
+
+CMD /bin/bash -c "marvin --help"
\ No newline at end of file
diff --git a/build/docker-r/Dockerfile b/build/docker-r/Dockerfile
new file mode 100644
index 0000000..f732aec
--- /dev/null
+++ b/build/docker-r/Dockerfile
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_TAG=latest
+FROM marvin-base:${BASE_TAG}
+
+##############################################################
+# Create all folders needed
+##############################################################
+
+
+##############################################################
+# Install the system dependencies for default R installation
+##############################################################
+
+RUN apt install -y apt-transport-https &&\
+ apt update && \
+ apt install -y python2.7-dev && \
+ apt install -y python-pip && \
+ apt install -y ipython && \
+ pip install --upgrade pip==9.0.1 && \
+ apt install -y libzmq3-dev libcurl4-openssl-dev libssl-dev && \
+ pip install jupyter
+
+
+RUN apt install -y r-base && \
+ su -c "R -e \"install.packages(c('repr', 'IRdisplay', 'IRkernel'), type = 'source', repos='http://cran.rstudio.com/')\"" && \
+ su -c "R -e \"IRkernel::installspec(user = FALSE)\""
+
+
+##############################################################
+# Copy and Install the marvin engine inside virtualenv
+##############################################################
+
+### TODO - Uncomment this block once we have the common lib for R tested and we can run engines in R
+
+# adds the package containing the user-generated engine
+#ADD generated-engine $MARVIN_ENGINE_HOME
+
+# adds the freshly built engine server jar
+#ADD marvin-engine-executor-assembly.jar $MARVIN_DATA_PATH
+
+##############################################################
+# Starts the jupyter http server
+##############################################################
+
+EXPOSE 8888
+
+USER marvin
+
+CMD /bin/bash -c "jupyter notebook --ip=0.0.0.0 --notebook-dir=$MARVIN_HOME"
\ No newline at end of file
diff --git a/build/marvin-engine-executor-assembly.jar b/build/marvin-engine-executor-assembly.jar
new file mode 100644
index 0000000..ff8a8f4
--- /dev/null
+++ b/build/marvin-engine-executor-assembly.jar
Binary files differ
diff --git a/build/python-toolbox.tgz b/build/python-toolbox.tgz
new file mode 100644
index 0000000..86b60d1
--- /dev/null
+++ b/build/python-toolbox.tgz
Binary files differ
diff --git a/public-engines/image-classification-engine/Makefile b/public-engines/image-classification-engine/Makefile
index cfbedd5..0768697 100644
--- a/public-engines/image-classification-engine/Makefile
+++ b/public-engines/image-classification-engine/Makefile
@@ -1,4 +1,4 @@
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/image-classification-engine/README.md b/public-engines/image-classification-engine/README.md
index 8fd12a1..4425eb1 100644
--- a/public-engines/image-classification-engine/README.md
+++ b/public-engines/image-classification-engine/README.md
@@ -42,4 +42,5 @@
### Contact
-danilo.nunes@b2wdigital.com
+name: danilo.nunes
+email: dev@marvin.apache.org
\ No newline at end of file
diff --git a/public-engines/image-classification-engine/docs.yaml b/public-engines/image-classification-engine/docs.yaml
index 676069a..ae0c71c 100644
--- a/public-engines/image-classification-engine/docs.yaml
+++ b/public-engines/image-classification-engine/docs.yaml
@@ -4,8 +4,8 @@
title: marvin_image_classification_engine API Doc
contact:
name: mantainer Daniel Takabayashi
- email: daniel.takabayashi@b2wdigital.com
- url:
+ email: dev@marvin.apache.org
+ url: https://marvin.apache.org
license:
name: Apache License 2.0
servers:
diff --git a/public-engines/image-classification-engine/htmlcov/marvin_image_classification_engine_prediction_feedback_py.html b/public-engines/image-classification-engine/htmlcov/marvin_image_classification_engine_prediction_feedback_py.html
index 3cfd58a..2c8c9db 100644
--- a/public-engines/image-classification-engine/htmlcov/marvin_image_classification_engine_prediction_feedback_py.html
+++ b/public-engines/image-classification-engine/htmlcov/marvin_image_classification_engine_prediction_feedback_py.html
@@ -117,7 +117,7 @@
<p id="t1" class="pln"><span class="com">#!/usr/bin/env python</span><span class="strut"> </span></p>
<p id="t2" class="pln"><span class="com"># coding=utf-8</span><span class="strut"> </span></p>
<p id="t3" class="pln"><span class="strut"> </span></p>
-<p id="t4" class="pln"><span class="com"># Copyright [2017] [B2W Digital]</span><span class="strut"> </span></p>
+<p id="t4" class="pln"><span class="com"># Copyright [2019] [Apache Software Foundation]</span><span class="strut"> </span></p>
<p id="t5" class="pln"><span class="com">#</span><span class="strut"> </span></p>
<p id="t6" class="pln"><span class="com"># Licensed under the Apache License, Version 2.0 (the "License");</span><span class="strut"> </span></p>
<p id="t7" class="pln"><span class="com"># you may not use this file except in compliance with the License.</span><span class="strut"> </span></p>
diff --git a/public-engines/image-classification-engine/marvin_image_classification_engine/prediction/feedback.py b/public-engines/image-classification-engine/marvin_image_classification_engine/prediction/feedback.py
index bed51c1..4f926b1 100644
--- a/public-engines/image-classification-engine/marvin_image_classification_engine/prediction/feedback.py
+++ b/public-engines/image-classification-engine/marvin_image_classification_engine/prediction/feedback.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/image-classification-engine/setup.py b/public-engines/image-classification-engine/setup.py
index e028d22..8afc4fe 100644
--- a/public-engines/image-classification-engine/setup.py
+++ b/public-engines/image-classification-engine/setup.py
@@ -57,8 +57,8 @@
description='Image Classification',
long_description=open(join(dirname(__file__), 'README.md')).read(),
author='Marvin AI Researcher',
- maintainer='B2W Labs Team',
- maintainer_email='@b2wdigital.com',
+ maintainer='Apache Software Foundation',
+ maintainer_email='dev@marvin.apache.org',
packages=find_packages(exclude=('tests', 'tests.*')),
include_package_data=True,
zip_safe=False,
@@ -86,7 +86,7 @@
'tensorflow==1.3.0',
'h5py==2.7.1',
],
- dependency_links=['git+https://github.com/marvin-ai/marvin-python-toolbox.git/@master#egg=marvin_python_toolbox-0'],
+ dependency_links=['git+https://github.com/apache/incubator-marvin@master#subdirectory=python-toolbox#egg=marvin_python_toolbox-0'],
tests_require=REQUIREMENT_TESTS,
extras_require={
'testing': REQUIREMENT_TESTS,
diff --git a/public-engines/iris-automl/.bumpversion.cfg b/public-engines/iris-automl/.bumpversion.cfg
new file mode 100644
index 0000000..102f3a3
--- /dev/null
+++ b/public-engines/iris-automl/.bumpversion.cfg
@@ -0,0 +1,5 @@
+[bumpversion]
+current_version = 0.0.1
+
+[bumpversion:file:marvin_iris_automl/VERSION]
+[bumpversion:file:README.md]
\ No newline at end of file
diff --git a/public-engines/iris-automl/.coveragerc b/public-engines/iris-automl/.coveragerc
new file mode 100644
index 0000000..6ebe78e
--- /dev/null
+++ b/public-engines/iris-automl/.coveragerc
@@ -0,0 +1,22 @@
+[run]
+omit = tests/*
+branch = True
+
+[report]
+exclude_lines =
+ pragma: no cover
+
+ def __repr__
+ if self\.debug
+
+ raise AssertionError
+ raise NotImplementedError
+
+ if 0:
+ if __name__ == .__main__.:
+
+[html]
+directory = coverage_report
+
+[xml]
+output = coverage_report.xml
\ No newline at end of file
diff --git a/public-engines/iris-automl/.gitignore b/public-engines/iris-automl/.gitignore
new file mode 100644
index 0000000..05dd2a2
--- /dev/null
+++ b/public-engines/iris-automl/.gitignore
@@ -0,0 +1,16 @@
+.cache
+.eggs
+.tox
+.testmondata
+.coverage
+.coverage.*
+coverage_report.xml
+coverage_report
+*.egg
+*.egg-info
+*.pyc
+tests/__pycache__
+.DS_Store
+.packages
+.profiling
+notebooks/data
\ No newline at end of file
diff --git a/public-engines/iris-automl/CHANGES.md b/public-engines/iris-automl/CHANGES.md
new file mode 100644
index 0000000..4835f6f
--- /dev/null
+++ b/public-engines/iris-automl/CHANGES.md
@@ -0,0 +1,5 @@
+## Changes log
+
+### 0.0.1
+
+ - initial version
\ No newline at end of file
diff --git a/public-engines/iris-automl/Dockerfile b/public-engines/iris-automl/Dockerfile
new file mode 100644
index 0000000..f117241
--- /dev/null
+++ b/public-engines/iris-automl/Dockerfile
@@ -0,0 +1,113 @@
+############################################################
+FROM python:2-alpine3.10
+# To use python3 comment the line above and uncomment
+# the line bellow.
+#FROM python:3-alpine3.10
+############################################################
+
+MAINTAINER dev@marvin.apache.org
+
+ENV SLEEP_MILLIS 0
+
+USER root
+
+##############################################################
+# Define all environment variables to be used
+##############################################################
+
+ENV MARVIN_HOME=/opt/marvin
+ENV MARVIN_DATA_PATH=/marvin-data
+ENV MARVIN_ENGINE_HOME=$MARVIN_HOME/engine
+ENV MARVIN_ENGINE_ENV=marvin-engine-env
+ENV WORKON_HOME=$MARVIN_HOME/.virtualenvs
+ENV SPARK_HOME=/opt/spark
+ENV SPARK_CONF_DIR=$SPARK_HOME/conf
+ENV HADOOP_CONF_DIR=$SPARK_CONF_DIR
+ENV YARN_CONF_DIR=$SPARK_CONF_DIR
+
+
+
+##############################################################
+# Create all folders needed
+##############################################################
+
+RUN mkdir -p $MARVIN_HOME && \
+ mkdir -p $MARVIN_DATA_PATH && \
+ mkdir -p $MARVIN_ENGINE_HOME && \
+ mkdir -p /var/log/marvin/engines && \
+ mkdir -p /var/run/marvin/engines && \
+##############################################################
+# Install the system dependencies for default installation
+##############################################################
+ apk add --no-cache g++ openssl-dev openjdk11-jre-headless bash && \
+ apk add --no-cache --virtual .build-deps make \
+ git \
+ wget \
+ libsass-dev \
+ openblas-dev \
+ libffi-dev \
+ libxml2-dev \
+ libxslt-dev \
+ libpng-dev \
+ freetype-dev \
+ cyrus-sasl-dev
+##############################################################
+# Install Apache Spark
+#
+# Uncomment if you are using spark, note that is needed the
+# spark configuration files to the think works correctly.
+##############################################################
+#
+# RUN wget -O /tmp/spark-2.1.1-bin-hadoop2.6.tgz https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz && \
+# tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/ && \
+# ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark
+##############################################################
+
+RUN mkdir -p $SPARK_CONF_DIR
+
+##############################################################
+# <CUSTOM ENGINE INSTALLATION PROCEDURE HERE> #
+##############################################################
+
+
+##############################################################
+# Copy and Install the marvin engine
+##############################################################
+
+RUN /bin/bash -c "pip install virtualenvwrapper && \
+ cd $MARVIN_ENGINE_HOME && \
+ source /usr/local/bin/virtualenvwrapper.sh && \
+ mkvirtualenv $MARVIN_ENGINE_ENV"
+
+ADD build/engine.tar $MARVIN_ENGINE_HOME
+
+ADD build/marvin-engine-executor-assembly.jar $MARVIN_DATA_PATH
+
+RUN /bin/bash -c "source /usr/local/bin/virtualenvwrapper.sh && \
+ workon $MARVIN_ENGINE_ENV && \
+ cd $MARVIN_ENGINE_HOME && \
+ pip install --no-cache numpy && \
+ pip install --no-cache scipy && \
+ pip install --no-cache pandas && \
+ pip install --no-cache matplotlib && \
+ pip install --no-cache cython && \
+ pip install --no-cache scikit-learn && \
+ pip install --no-cache Fabric && \
+ pip install --no-cache marvin-python-toolbox && \
+ pip install . "
+##############################################################
+# Uninstalling unnecessary software and cleaning cache
+##############################################################
+RUN rm -rf /root/.cache && \
+ apk del .build-deps
+
+##############################################################
+# Starts the engine http server
+##############################################################
+
+EXPOSE 8000
+
+CMD /bin/bash -c "source /usr/local/bin/virtualenvwrapper.sh && \
+ workon $MARVIN_ENGINE_ENV && \
+ cd $MARVIN_ENGINE_HOME && \
+ marvin engine-httpserver -h 0.0.0.0 -p 8000"
\ No newline at end of file
diff --git a/public-engines/iris-automl/INSTALL b/public-engines/iris-automl/INSTALL
new file mode 100644
index 0000000..fccdaf8
--- /dev/null
+++ b/public-engines/iris-automl/INSTALL
@@ -0,0 +1 @@
+REPLACE: Add here the detailed instructions to install this project
\ No newline at end of file
diff --git a/public-engines/iris-automl/LICENSE b/public-engines/iris-automl/LICENSE
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/public-engines/iris-automl/LICENSE
diff --git a/public-engines/iris-automl/MANIFEST.in b/public-engines/iris-automl/MANIFEST.in
new file mode 100644
index 0000000..2d283ba
--- /dev/null
+++ b/public-engines/iris-automl/MANIFEST.in
@@ -0,0 +1,9 @@
+include CHANGES.md
+include INSTALL
+include LICENSE
+include MANIFEST.in
+include README.md
+include marvin_iris_automl/VERSION
+recursive-include notebooks *
+prune notebooks/build
+recursive-include tests *
\ No newline at end of file
diff --git a/public-engines/iris-automl/Makefile b/public-engines/iris-automl/Makefile
new file mode 100644
index 0000000..a1a8399
--- /dev/null
+++ b/public-engines/iris-automl/Makefile
@@ -0,0 +1,81 @@
+.PHONY: help marvin marvin-automl marvin-prod update clean-pyc clean-build clean-reports clean-deps clean docker-build docker-push docker-run
+
+DOCKER_VERSION?=0.00.01
+DOCKER_REGISTRY_ADRESS?=docker.registry.io
+MARVIN_DATA_PATH?=$(HOME)/marvin/data
+MARVIN_ENGINE_NAME?=marvin_iris_automl
+MARVIN_TOOLBOX_VERSION?=0.0.5
+
+help:
+ @echo " marvin"
+ @echo " Prepare project to be used as a marvin package."
+ @echo " marvin-prod"
+ @echo " Prepare project to be used in production environment."
+ @echo " marvin-automl"
+ @echo " Prepare project to be used as a marvin-automl package."
+ @echo " update"
+ @echo " Reinstall requirements and setup.py dependencies."
+ @echo " clean"
+ @echo " Remove all generated artifacts."
+ @echo " clean-pyc"
+ @echo " Remove python artifacts."
+ @echo " clean-build"
+ @echo " Remove build artifacts."
+ @echo " clean-reports"
+ @echo " Remove coverage reports."
+ @echo " clean-deps"
+ @echo " Remove marvin setup.py dependencies."
+ @echo " docker-build"
+ @echo " Runs the docker build command with marvin env default parameters."
+ @echo " docker-push"
+ @echo " Runs the docker push command with marvin env default parameters."
+ @echo " docker-run"
+ @echo " Runs the docker run command with marvin env default parameters."
+
+marvin:
+ pip install -e ".[testing]"
+ marvin --help
+marvin-automl:
+ pip install -e ".[testing]"
+ bash install_automl.sh
+ marvin --help
+marvin-prod:
+ pip install .
+ marvin --help
+
+update:
+ pip install -e . -U
+
+clean-pyc:
+ find . -name '*.pyc' -exec rm -f {} +
+ find . -name '*.pyo' -exec rm -f {} +
+ find . -name '*~' -exec rm -f {} +
+
+clean-build:
+ rm -rf *.egg-info
+ rm -rf .cache
+ rm -rf .eggs
+ rm -rf dist
+ rm -rf build
+
+clean-reports:
+ rm -rf coverage_report/
+ rm -f coverage.xml
+ rm -f .coverage
+
+clean-deps:
+ pip freeze | grep -v "^-e" | xargs pip uninstall -y
+
+clean: clean-build clean-pyc clean-reports clean-deps
+
+docker-build: clean-build
+ mkdir -p build
+ tar -cf build/engine.tar --exclude=*.log --exclude=*.pkl --exclude='build' --exclude='notebooks' --exclude=*.tar *
+ cp -f $(MARVIN_DATA_PATH)/marvin-engine-executor-assembly-$(MARVIN_TOOLBOX_VERSION).jar build/marvin-engine-executor-assembly.jar
+ sudo docker build -t $(DOCKER_REGISTRY_ADRESS)/$(MARVIN_ENGINE_NAME):$(DOCKER_VERSION) .
+
+docker-run:
+ sudo docker run --name=marvin-$(MARVIN_ENGINE_NAME)-$(DOCKER_VERSION) --mount type=bind,source=$(MARVIN_DATA_PATH),destination=/marvin-data -p 8000:8000 $(DOCKER_REGISTRY_ADRESS)/$(MARVIN_ENGINE_NAME):$(DOCKER_VERSION)
+
+docker-push:
+ sudo docker push $(DOCKER_REGISTRY_ADRESS)/$(MARVIN_ENGINE_NAME):$(DOCKER_VERSION)
\ No newline at end of file
diff --git a/public-engines/iris-automl/README.md b/public-engines/iris-automl/README.md
new file mode 100644
index 0000000..5f7fc91
--- /dev/null
+++ b/public-engines/iris-automl/README.md
@@ -0,0 +1,151 @@
+# iris_automl_engine v0.0.1
+
+## Overview
+
+Iris Species classification engine from Kaggle with auto-sklearn (https://automl.github.io/auto-sklearn) and tpot (http://epistasislab.github.io/tpot/)
+
+
+## Requirements
+
+ - Python 3.7
+ - Numpy 1.11.0 or higher
+ - tpot
+ - auto-sklearn
+ - swig (Dependence of auto-sklearn)
+ - dialog (Dependence of install process)
+
+
+## Installation
+
+Create a new env with all dependences, replace PATH for iris-automl-engine path:
+
+ marvin engine-generateenv PATH
+
+Select tpot and auto-sklearn in dialog window after run engine-generateenv.
+
+Use the Marvin toolbox to provision, deploy and start the remote HTTP server.
+
+Edit the `marvin.ini` file, setting the options within the
+`ssh_deployment` section:
+
+1. `host`: the host IP address or name where the engine should be deployed. You
+can enable multi-host deployment using `,` to separate hosts
+2. `port`: the SSH connection port
+3. `user`: the SSH connection username. Currently, only a single user is
+supported. This user should be capable of *passwordless sudo*, although it can
+use password for the SSH connection
+
+Next, ensure that the remotes servers are provisioned (all required software
+are installed):
+
+ marvin engine-deploy --provision
+
+Next, package your engine:
+
+ marvin engine-deploy --package
+
+This will create a compressed archive containing your engine code under the
+`.packages` directory.
+
+Next, deploy your engine to remotes servers:
+
+ marvin engine-deploy
+
+By default, a dependency clean will be executed at each deploy. You can skip it
+using:
+
+ marvin engine-deploy --skip-clean
+
+Next, you can start the HTTP server in the remotes servers:
+
+ marvin engine-httpserver-remote start
+
+You can check if the HTTP server is running:
+
+ marvin engine-httpserver-remote status
+
+And stop it:
+
+ marvin engine-httpserver-remote stop
+
+After starting, you can test it by making a HTTP request to any endpoint, like:
+
+ curl -v http://example.com/predictor/health
+
+Under the hood, this engine uses Fabric to define provisioning and deployment
+process. Check the `fabfile.py` for more information. You can add new tasks or
+edit existing ones to match your provisioning and deployment pipeline.
+
+## Development
+
+### Getting started
+
+First, create a new virtualenv
+
+```
+mkvirtualenv marvin_iris_automl_engine_env
+```
+
+Now install the development dependencies
+
+```
+make marvin
+```
+
+You are now ready to code.
+
+
+### Adding new dependencies
+
+It\`s very important. All development dependencies should be added to `setup.py`.
+
+### Running tests
+
+This project uses *[py.test](http://pytest.org/)* as test runner and *[Tox](https://tox.readthedocs.io)* to manage virtualenvs.
+
+To run all tests use the following command
+
+```
+marvin test
+```
+
+To run specific test
+
+```
+marvin test tests/test_file.py::TestClass::test_method
+```
+
+
+### Writting documentation
+
+The project documentation is written using *[Jupyter](http://jupyter.readthedocs.io/)* notebooks.
+You can start the notebook server from the command line by running the following command
+
+```
+marvin notebook
+```
+
+Use notebooks to demonstrate how to use the lib features. It can also be useful to show some use cases.
+
+
+### Bumping version
+
+```
+marvin pkg-bumpversion [patch|minor|major]
+git add . && git commit -m "Bump version"
+```
+
+
+### Tagging version
+
+```
+marvin pkg-createtag
+git push origin master --follow-tags
+```
+
+
+### Logging
+
+The default log level is set to _WARNING_. You can change the log level at runtime setting another value to one of the following environment variable: `MARVIN_IRIS_AUTOML_ENGINE_LOG_LEVEL` or `LOG_LEVEL`. The available values are _CRITICAL_, _ERROR_, _WARNING_, _INFO_ and _DEBUG_.
+
+Be careful using `LOG_LEVEL`, it may affect another lib.
diff --git a/public-engines/iris-automl/docs.yaml b/public-engines/iris-automl/docs.yaml
new file mode 100644
index 0000000..d64c439
--- /dev/null
+++ b/public-engines/iris-automl/docs.yaml
@@ -0,0 +1,567 @@
+openapi: "3.0.0"
+info:
+ version: 0.0.1
+ title: marvin_iris_automl API Doc
+ contact:
+ name: mantainer Marvin AI Community
+ email: dev@marvin.apache.org
+ url: https://marvin.apache.org
+ license:
+ name: Apache License 2.0
+servers:
+ - url: http://localhost:8000
+ - url: http://0.0.0.0:8000
+tags:
+ - name: Docker
+ description: For Docker users, please use "make docker-build" and "make docker-run" commands in your engine virtualenv to start the server
+ - name: Acquisitor
+ description: Setup the initial_dataset with all cleaned data necessary to build your dataset in the next action
+ - name: Tpreparator
+ description: Setup the dataset with the transformed data that is compatible with the algorithm used to build the model in the next action
+ - name: Trainer
+ description: Setup the model with the result of the algorithm used to training
+ - name: Evaluator
+ description: Setup the metrics with the result of the algorithms used to test the model
+ - name: Predictor
+ description: Return the predicted value in a json parsable object format
+ - name: Feedback
+ description: Receive feedback message, user can manipulate this message for any use
+ - name: Pipeline
+ description: Perform all batch actions in your right order
+paths:
+ /acquisitor/health:
+ get:
+ summary: Get acquisitor's service health
+ operationId: getAcquisitorHealth
+ tags:
+ - Acquisitor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /acquisitor/status:
+ get:
+ summary: Get acquisitor's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getAcquisitorStatus
+ tags:
+ - Acquisitor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /acquisitor:
+ post:
+ summary: Run acquisitor
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: acquisitor
+ tags:
+ - Acquisitor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator/health:
+ get:
+ summary: Get trainer preparator's service health
+ operationId: getTPreparatortorHealth
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator/status:
+ get:
+ summary: Get trainer preparator's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getTPreparatorStatus
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator/reload?protocol=:
+ put:
+ summary: Reload artifact for trainer preparator
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadTPreparator
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator:
+ post:
+ summary: Run trainer preparator
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: tpreparator
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer/health:
+ get:
+ summary: Get trainer's service health
+ operationId: getTrainerHealth
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer/status:
+ get:
+ summary: Get trainer's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getTrainerStatus
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer/reload?protocol=:
+ put:
+ summary: Reload artifact for trainer
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadTrainer
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer:
+ post:
+ summary: Run trainer
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: trainer
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/health:
+ get:
+ summary: Get evaluator's service health
+ operationId: getEvaluatorHealth
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/status:
+ get:
+ summary: Get evaluator's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getEvaluatorStatus
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/metrics?protocol=:
+ get:
+ summary: Get metrics's value
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: Metrics protocol value
+ operationId: getMetrics
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/reload?protocol=:
+ put:
+ summary: Reload artifact for evaluator
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: reloadEvaluator
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator:
+ post:
+ summary: Run evaluator
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: evaluator
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor/health:
+ get:
+ summary: Get predictor's service health
+ operationId: getPredictorHealth
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor/status:
+ get:
+ summary: Get predictor's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getPredictorStatus
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor/reload?protocol=:
+ put:
+ summary: Reload artifact for predictor
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadPredictor
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor:
+ post:
+ summary: Run predictor
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: predictor
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback/health:
+ get:
+ summary: Get feedback's service health
+ operationId: getFeedbackHealth
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback/status:
+ get:
+ summary: Get feedback's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getFeedbackStatus
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback/reload?protocol=:
+ put:
+ summary: Reload artifact for feedback
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadFeedback
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback:
+ post:
+ summary: Run feedback
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: feedback
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /pipeline:
+ post:
+ summary: Do all batch actions (from Acquisitor to Evaluator)
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: pipeline
+ tags:
+ - Pipeline
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
\ No newline at end of file
diff --git a/public-engines/iris-automl/engine.messages b/public-engines/iris-automl/engine.messages
new file mode 100644
index 0000000..3d8c0f5
--- /dev/null
+++ b/public-engines/iris-automl/engine.messages
@@ -0,0 +1 @@
+[[1,2,3,4]]
diff --git a/public-engines/iris-automl/engine.metadata b/public-engines/iris-automl/engine.metadata
new file mode 100644
index 0000000..71ca401
--- /dev/null
+++ b/public-engines/iris-automl/engine.metadata
@@ -0,0 +1,62 @@
+{
+ "name": "marvin_iris_automl",
+ "version": "v0.0.1",
+ "engineType": "python",
+ "artifactsRemotePath": "/tmp/marvin",
+ "artifactManagerType": "FS",
+ "onlineActionTimeout": 1000,
+ "metricsTimeout": 10000,
+ "healthCheckTimeout": 2000,
+ "reloadTimeout": 600000,
+ "batchActionTimeout": 600000,
+ "pipelineActions": ["acquisitor", "tpreparator", "trainer", "evaluator"],
+ "actions": [{
+ "name": "acquisitor",
+ "actionType": "batch",
+ "port": 50051,
+ "host": "localhost",
+ "artifactsToPersist": ["initialdataset"],
+ "artifactsToLoad": [],
+ "pipeline": []
+ }, {
+ "name": "tpreparator",
+ "actionType": "batch",
+ "port": 50052,
+ "host": "localhost",
+ "artifactsToPersist": ["dataset"],
+ "artifactsToLoad": ["initialdataset"],
+ "pipeline": []
+ }, {
+ "name": "trainer",
+ "actionType": "batch",
+ "port": 50053,
+ "host": "localhost",
+ "artifactsToPersist": ["model"],
+ "artifactsToLoad": ["dataset"],
+ "pipeline": []
+ }, {
+ "name": "evaluator",
+ "actionType": "batch",
+ "port": 50054,
+ "host": "localhost",
+ "artifactsToPersist": ["metrics"],
+ "artifactsToLoad": ["dataset", "model"],
+ "pipeline": []
+ }, {
+ "name": "predictor",
+ "actionType": "online",
+ "port": 50055,
+ "host": "localhost",
+ "artifactsToPersist": [],
+ "artifactsToLoad": ["model", "metrics"],
+ "pipeline": ["ppreparator"]
+ }, {
+ "name": "feedback",
+ "actionType": "online",
+ "port": 50056,
+ "host": "localhost",
+ "artifactsToPersist": [],
+ "artifactsToLoad": [],
+ "pipeline": []
+ }]
+}
\ No newline at end of file
diff --git a/public-engines/iris-automl/engine.params b/public-engines/iris-automl/engine.params
new file mode 100644
index 0000000..410f586
--- /dev/null
+++ b/public-engines/iris-automl/engine.params
@@ -0,0 +1,3 @@
+{
+ "PARAM_1" : "VALUE_OF_PARAM_1"
+}
\ No newline at end of file
diff --git a/public-engines/iris-automl/fabfile.py b/public-engines/iris-automl/fabfile.py
new file mode 100644
index 0000000..851959f
--- /dev/null
+++ b/public-engines/iris-automl/fabfile.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+from fabric.api import env
+from fabric.api import run
+from fabric.api import execute
+from fabric.api import cd
+from fabric.api import local
+from fabric.api import put
+from fabric.api import sudo
+from fabric.state import output
+from marvin_python_toolbox import __version__ as TOOLBOX_VERSION
+from marvin_python_toolbox.common.config import Config
+
+_host = Config.get("host", section="ssh_deployment").split(",")
+_port = Config.get("port", section="ssh_deployment")
+_user = Config.get("user", section="ssh_deployment")
+
+for h in _host:
+ env.hosts.append("{user}@{host}:{port}".format(user=_user, host=h, port=_port))
+
+output["everything"] = False
+output["running"] = True
+
+env.package = "marvin_iris_automl"
+env.margin_engine_executor_prefix = "/opt/marvin/engine-executor"
+env.margin_engine_executor_jar = "marvin-engine-executor-assembly-{version}.jar".format(version=TOOLBOX_VERSION)
+env.marvin_engine_executor_path = env.margin_engine_executor_prefix + "/" + env.margin_engine_executor_jar
+
+
+def install_oracle_jdk():
+ sudo("add-apt-repository ppa:webupd8team/java -y")
+ sudo("apt-get -qq update")
+ run("echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections")
+ run("echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections")
+ sudo("apt-get install -y oracle-java8-installer")
+
+
+def install_virtualenvwrapper():
+ run("pip install virtualenvwrapper")
+ run("echo 'export WORKON_HOME=${HOME}/.virtualenvs' >> ${HOME}/.profile")
+ run("echo 'source /usr/local/bin/virtualenvwrapper.sh' >> ${HOME}/.profile")
+
+
+def install_apache_spark():
+ run("curl https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz -o /tmp/spark-2.1.1-bin-hadoop2.6.tgz")
+ sudo("tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/")
+ sudo("ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark")
+ run("echo 'export SPARK_HOME=/opt/spark' >> ${HOME}/.profile")
+
+
+def install_required_packages():
+ sudo("apt-get update -y")
+ sudo("apt-get install -y git")
+ sudo("apt-get install -y wget")
+ sudo("apt-get install -y python2.7-dev")
+ sudo("apt-get install -y python-pip")
+ sudo("apt-get install -y ipython")
+ sudo("apt-get install -y libffi-dev")
+ sudo("apt-get install -y libssl-dev")
+ sudo("apt-get install -y libxml2-dev")
+ sudo("apt-get install -y libxslt1-dev")
+ sudo("apt-get install -y libpng12-dev")
+ sudo("apt-get install -y libfreetype6-dev")
+ sudo("apt-get install -y python-tk")
+ sudo("apt-get install -y libsasl2-dev")
+ sudo("apt-get install -y python-pip")
+ sudo("apt-get install -y graphviz")
+ sudo("pip install --upgrade pip")
+
+
+def install_marvin_engine_executor():
+ sudo("mkdir -p {prefix}".format(prefix=env.margin_engine_executor_prefix))
+ with cd("{prefix}".format(prefix=env.margin_engine_executor_prefix)):
+ sudo("wget https://s3.amazonaws.com/marvin-engine-executor/{jar}".format(jar=env.margin_engine_executor_jar))
+
+
+def create_marvin_engines_prefix():
+ sudo("mkdir -p /opt/marvin/engines")
+ sudo("chown {user}:{user} /opt/marvin/engines".format(user=env.user))
+ sudo("mkdir -p /var/log/marvin/engines")
+ sudo("chown {user}:{user} /var/log/marvin/engines".format(user=env.user))
+ sudo("mkdir -p /var/run/marvin/engines")
+ sudo("chown {user}:{user} /var/run/marvin/engines".format(user=env.user))
+
+
+def configure_marvin_environment():
+ run("echo 'export MARVIN_HOME=${HOME}/marvin' >> ${HOME}/.profile")
+ run("echo 'export MARVIN_DATA_PATH=${MARVIN_HOME}/data' >> ${HOME}/.profile")
+ run("mkdir -p ${MARVIN_HOME}")
+ run("mkdir -p ${MARVIN_DATA_PATH}")
+
+
+def provision():
+ execute(install_required_packages)
+ execute(install_virtualenvwrapper)
+ execute(install_oracle_jdk)
+ execute(install_apache_spark)
+ execute(install_marvin_engine_executor)
+ execute(create_marvin_engines_prefix)
+ execute(configure_marvin_environment)
+
+
+def package(version):
+ package = env.package
+ local("mkdir -p .packages")
+ local("tar czvf .packages/{package}-{version}.tar.gz --exclude='.packages' .".format(
+ package=package, version=version))
+
+
+def deploy(version, skip_clean=False):
+ execute(engine_stop)
+ package = env.package
+ put(local_path=".packages/{package}-{version}.tar.gz".format(
+ package=package, version=version), remote_path="/tmp/")
+ run("mkdir -p /opt/marvin/engines/{package}/{version}".format(
+ package=package, version=version))
+ with cd("/opt/marvin/engines/{package}/{version}".format(
+ package=package, version=version)):
+ run("tar xzvf /tmp/{package}-{version}.tar.gz".format(
+ package=package, version=version))
+ with cd("/opt/marvin/engines/{package}".format(package=package)):
+ symlink_exists = run("stat current", quiet=True).succeeded
+ if (symlink_exists):
+ run("rm current")
+ run("ln -s {version} current".format(version=version))
+ with cd("/opt/marvin/engines/{package}/current".format(package=package)):
+ run("mkvirtualenv {package}_env".format(package=package))
+ run("setvirtualenvproject")
+ if skip_clean:
+ run("workon {package}_env && make marvin".format(
+ package=package))
+ else:
+ run("workon {package}_env && make clean && make marvin".format(
+ package=package))
+ execute(engine_start)
+
+
+def engine_start(http_host, http_port):
+ package = env.package
+
+ command = (
+ "workon {package}_env &&"
+ " (marvin engine-httpserver"
+ " -h {http_host}"
+ " -p {http_port}"
+ " -e {executor}"
+ " 1> /var/log/marvin/engines/{package}.out"
+ " 2> /var/log/marvin/engines/{package}.err"
+ " & echo $! > /var/run/marvin/engines/{package}.pid)"
+ ).format(
+ package=package,
+ http_host=http_host,
+ http_port=http_port,
+ executor=env.marvin_engine_executor_path
+ )
+
+ with cd("/opt/marvin/engines/{package}/current".format(package=package)):
+ run(command, pty=False)
+
+
+def engine_stop():
+ package = env.package
+
+ pid_file_exists = run("cat /var/run/marvin/engines/{package}.pid".format(
+ package=package), quiet=True)
+ if pid_file_exists.succeeded:
+ with cd("/opt/marvin/engines/{package}/current".format(package=package)):
+ children_pids = run("ps --ppid $(cat /var/run/marvin/engines/{package}.pid) -o pid --no-headers |xargs echo".format(
+ package=package))
+ run("kill $(cat /var/run/marvin/engines/{package}.pid) {children_pids}".format(
+ package=package, children_pids=children_pids))
+ run("rm /var/run/marvin/engines/{package}.pid".format(package=package))
+
+
+def engine_status():
+ package = env.package
+ pid_file_exists = run("cat /var/run/marvin/engines/{package}.pid".format(
+ package=package), quiet=True)
+ if pid_file_exists.succeeded:
+ is_running = run("ps $(cat /var/run/marvin/engines/{package}.pid)".format(package=package), quiet=True)
+ if is_running.succeeded:
+ print "Your engine is running :)"
+ else:
+ print "Your engine is not running :("
+ else:
+ print "Your engine is not running :("
\ No newline at end of file
diff --git a/public-engines/iris-automl/feedback.messages b/public-engines/iris-automl/feedback.messages
new file mode 100644
index 0000000..5ff7ec4
--- /dev/null
+++ b/public-engines/iris-automl/feedback.messages
@@ -0,0 +1,3 @@
+[{
+ "msg1": "Hello from marvin engine!"
+}]
\ No newline at end of file
diff --git a/public-engines/iris-automl/install_automl.sh b/public-engines/iris-automl/install_automl.sh
new file mode 100644
index 0000000..bc73021
--- /dev/null
+++ b/public-engines/iris-automl/install_automl.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+###############################################
+# Marvin AutoML installation Script #
+###############################################
+cmd=(dialog --title "Marvin AutoML" --separate-output --checklist "Select tools:" 22 76 16)
+options=(1 "auto-sklearn" off # any option can be set to default to "on"
+ 2 "h2o AutoML" off
+ 3 "TPOT" off)
+choices=$("${cmd[@]}" "${options[@]}" 2>&1 >/dev/tty)
+clear
+for choice in $choices
+do
+ case $choice in
+ 1)
+ echo "Instaling auto-sklearn..."
+ wget https://raw.githubusercontent.com/automl/auto-sklearn/master/requirements.txt \
+ | xargs -n 1 -L 1 pip install
+ pip install auto-sklearn
+ ;;
+ 2)
+ echo "Installing h2o..."
+ pip install requests
+ pip install tabulate
+ pip install scikit-learn
+ pip install http://h2o-release.s3.amazonaws.com/h2o/rel-yau/3/Python/h2o-3.26.0.3-py2.py3-none-any.whl
+ wget http://h2o-release.s3.amazonaws.com/h2o/rel-yau/3/h2o-3.26.0.3.zip
+ unzip h2o-3.26.0.3.zip
+ rm h2o-3.26.0.3.zip
+ ;;
+ 3)
+ echo "Installing TPOT..."
+ pip install tpot
+ ;;
+ esac
+done
\ No newline at end of file
diff --git a/public-engines/iris-automl/marvin.ini b/public-engines/iris-automl/marvin.ini
new file mode 100644
index 0000000..723eaee
--- /dev/null
+++ b/public-engines/iris-automl/marvin.ini
@@ -0,0 +1,11 @@
+[marvin]
+package = marvin_iris_automl
+type = automl-engine
+executor_url = https://s3.amazonaws.com/marvin-engine-executor/marvin-engine-executor-assembly-0.0.5.jar
+
+[ssh_deployment]
+# You can enable multi-host deployment like this
+# host = host1.com,host2.com,hostN.com
+host = host1.com
+port = 22
+user = marvin
\ No newline at end of file
diff --git a/public-engines/iris-automl/marvin_iris_automl/VERSION b/public-engines/iris-automl/marvin_iris_automl/VERSION
new file mode 100644
index 0000000..8a9ecc2
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/VERSION
@@ -0,0 +1 @@
+0.0.1
\ No newline at end of file
diff --git a/public-engines/iris-automl/marvin_iris_automl/__init__.py b/public-engines/iris-automl/marvin_iris_automl/__init__.py
new file mode 100644
index 0000000..e715c57
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/__init__.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os.path
+
+from .data_handler import *
+from .prediction import *
+from .training import *
+
+
+# Get package version number from "VERSION" file
+with open(os.path.join(os.path.dirname(__file__), 'VERSION'), 'rb') as f:
+ __version__ = f.read().decode('ascii').strip()
diff --git a/public-engines/iris-automl/marvin_iris_automl/_compatibility.py b/public-engines/iris-automl/marvin_iris_automl/_compatibility.py
new file mode 100644
index 0000000..b2e635d
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/_compatibility.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Compatibility module.
+
+Import this module to help to write code compatible with Python 2 and 3.
+"""
+
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import six
+
+__all__ = ['six']
+
+# Add here any code that have to differentiate between python 2 and 3.
diff --git a/public-engines/iris-automl/marvin_iris_automl/_logging.py b/public-engines/iris-automl/marvin_iris_automl/_logging.py
new file mode 100644
index 0000000..9e464a3
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/_logging.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Custom logging module.
+
+This module is responsible to manage log messages and log file.
+"""
+
+import sys
+import os
+import os.path
+import logging
+
+DEFAULT_LOG_LEVEL = logging.WARNING
+DEFAULT_LOG_DIR = '/tmp'
+
+
+class Logger(logging.getLoggerClass()):
+ """Custom logger class.
+
+ Use this class to customize the logger behavior or to intercept the
+ messages.
+ """
+ def error(self, msg, *args, **kwargs):
+ # Add here code to intercept the project error messages
+ super(Logger, self).error(msg, *args, **kwargs)
+
+ def critical(self, msg, *args, **kwargs):
+ # Add here code to intercept the project critical messages
+ super(Logger, self).critical(msg, *args, **kwargs)
+
+
+logging.setLoggerClass(Logger)
+
+
+def get_logger(name, namespace='marvin_iris_automl',
+ log_level=DEFAULT_LOG_LEVEL, log_dir=DEFAULT_LOG_DIR):
+ """Build a logger that outputs to a file and to the console,"""
+
+ log_level = (os.getenv('{}_LOG_LEVEL'.format(namespace.upper())) or
+ os.getenv('LOG_LEVEL', log_level))
+ log_dir = (os.getenv('{}_LOG_DIR'.format(namespace.upper())) or
+ os.getenv('LOG_DIR', log_dir))
+
+ logger = logging.getLogger('{}.{}'.format(namespace, name))
+ logger.setLevel(log_level)
+
+ formatter = logging.Formatter(
+ '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+ # Create a console stream handler
+ console_handler = logging.StreamHandler()
+ console_handler.setLevel(log_level)
+ console_handler.setFormatter(formatter)
+ logger.addHandler(console_handler)
+
+ try:
+ if log_dir:
+ log_path = os.path.abspath(log_dir)
+ log_filename = '{name}.{pid}.log'.format(
+ name=namespace, pid=os.getpid())
+
+ file_path = str(os.path.join(log_path, log_filename))
+
+ if not os.path.exists(log_path):
+ os.makedirs(log_path, mode=774)
+
+ # Create a file handler
+ file_handler = logging.FileHandler(file_path)
+ file_handler.setLevel(log_level)
+ file_handler.setFormatter(formatter)
+ logger.addHandler(file_handler)
+ except OSError as e:
+ logger.error('Could not create log file {file}: {error}'.format(
+ file=file_path, error=e.strerror))
+
+ return logger
diff --git a/public-engines/iris-automl/marvin_iris_automl/data_handler/__init__.py b/public-engines/iris-automl/marvin_iris_automl/data_handler/__init__.py
new file mode 100644
index 0000000..677dd17
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/data_handler/__init__.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+from .acquisitor_and_cleaner import AcquisitorAndCleaner
+from .training_preparator import TrainingPreparator
\ No newline at end of file
diff --git a/public-engines/iris-automl/marvin_iris_automl/data_handler/acquisitor_and_cleaner.py b/public-engines/iris-automl/marvin_iris_automl/data_handler/acquisitor_and_cleaner.py
new file mode 100644
index 0000000..1a8e51d
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/data_handler/acquisitor_and_cleaner.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""AcquisitorAndCleaner engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBaseDataHandler
+
+__all__ = ['AcquisitorAndCleaner']
+
+
+logger = get_logger('acquisitor_and_cleaner')
+
+
+class AcquisitorAndCleaner(EngineBaseDataHandler):
+
+ def __init__(self, **kwargs):
+ super(AcquisitorAndCleaner, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ from sklearn import datasets
+ iris = datasets.load_iris()
+ self.marvin_initial_dataset = iris
+
diff --git a/public-engines/iris-automl/marvin_iris_automl/data_handler/training_preparator.py b/public-engines/iris-automl/marvin_iris_automl/data_handler/training_preparator.py
new file mode 100644
index 0000000..cc63e91
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/data_handler/training_preparator.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""TrainingPreparator engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBaseDataHandler
+
+__all__ = ['TrainingPreparator']
+
+
+logger = get_logger('training_preparator')
+
+
+class TrainingPreparator(EngineBaseDataHandler):
+
+ def __init__(self, **kwargs):
+ super(TrainingPreparator, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ import sklearn.model_selection
+ X, y = self.marvin_initial_dataset.data, self.marvin_initial_dataset.target
+ X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, random_state=1)
+
+ self.marvin_dataset = {'train_X': X_train, 'train_y': y_train, 'test_X': X_test, 'test_y': y_test}
+
diff --git a/public-engines/iris-automl/marvin_iris_automl/prediction/__init__.py b/public-engines/iris-automl/marvin_iris_automl/prediction/__init__.py
new file mode 100644
index 0000000..e9b3c7c
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/prediction/__init__.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+from .prediction_preparator import PredictionPreparator
+from .predictor import Predictor
+from .feedback import Feedback
\ No newline at end of file
diff --git a/public-engines/iris-automl/marvin_iris_automl/prediction/feedback.py b/public-engines/iris-automl/marvin_iris_automl/prediction/feedback.py
new file mode 100644
index 0000000..085808b
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/prediction/feedback.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2019] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Feedback engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBasePrediction
+
+__all__ = ['Feedback']
+
+
+logger = get_logger('feedback')
+
+
+class Feedback(EngineBasePrediction):
+
+ def __init__(self, **kwargs):
+ super(Feedback, self).__init__(**kwargs)
+
+ def execute(self, input_message, params, **kwargs):
+ """
+ Receive feedback message, user can manipulate this message for any use.
+ Return "Done" to signal that the message is received and processed.
+ """
+ return {"message": "Done"}
\ No newline at end of file
diff --git a/public-engines/iris-automl/marvin_iris_automl/prediction/prediction_preparator.py b/public-engines/iris-automl/marvin_iris_automl/prediction/prediction_preparator.py
new file mode 100644
index 0000000..9745020
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/prediction/prediction_preparator.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""PredictionPreparator engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBasePrediction
+
+__all__ = ['PredictionPreparator']
+
+
+logger = get_logger('prediction_preparator')
+
+
+class PredictionPreparator(EngineBasePrediction):
+
+ def __init__(self, **kwargs):
+ super(PredictionPreparator, self).__init__(**kwargs)
+
+ def execute(self, input_message, params, **kwargs):
+ import numpy as np
+ input_message = np.array([input_message])
+
+ return input_message
diff --git a/public-engines/iris-automl/marvin_iris_automl/prediction/predictor.py b/public-engines/iris-automl/marvin_iris_automl/prediction/predictor.py
new file mode 100644
index 0000000..424eaf9
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/prediction/predictor.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Predictor engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBasePrediction
+
+__all__ = ['Predictor']
+
+
+logger = get_logger('predictor')
+
+
+class Predictor(EngineBasePrediction):
+
+ def __init__(self, **kwargs):
+ super(Predictor, self).__init__(**kwargs)
+
+ def execute(self, input_message, params, **kwargs):
+ from sklearn import datasets
+ iris = datasets.load_iris()
+ prediction = self.marvin_model.predict(input_message)
+ final_prediction = iris.target_names[prediction][0]
+
+ return final_prediction
diff --git a/public-engines/iris-automl/marvin_iris_automl/training/__init__.py b/public-engines/iris-automl/marvin_iris_automl/training/__init__.py
new file mode 100644
index 0000000..e1723b7
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/training/__init__.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+from .metrics_evaluator import MetricsEvaluator
+from .trainer import Trainer
\ No newline at end of file
diff --git a/public-engines/iris-automl/marvin_iris_automl/training/metrics_evaluator.py b/public-engines/iris-automl/marvin_iris_automl/training/metrics_evaluator.py
new file mode 100644
index 0000000..22f891a
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/training/metrics_evaluator.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""MetricsEvaluator engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBaseTraining
+
+__all__ = ['MetricsEvaluator']
+
+
+logger = get_logger('metrics_evaluator')
+
+
+class MetricsEvaluator(EngineBaseTraining):
+
+ def __init__(self, **kwargs):
+ super(MetricsEvaluator, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ from sklearn import metrics
+ predictions = self.marvin_model.predict(self.marvin_dataset['test_X'])
+ self.marvin_metrics = metrics.accuracy_score(self.marvin_dataset['test_y'], predictions)
+
diff --git a/public-engines/iris-automl/marvin_iris_automl/training/trainer.py b/public-engines/iris-automl/marvin_iris_automl/training/trainer.py
new file mode 100644
index 0000000..7eee0e9
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/training/trainer.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Trainer engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBaseTraining
+
+__all__ = ['Trainer']
+
+
+logger = get_logger('trainer')
+
+
+class Trainer(EngineBaseTraining):
+
+ def __init__(self, **kwargs):
+ super(Trainer, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ import autosklearn.classification
+
+ automl = autosklearn.classification.AutoSklearnClassifier(
+ time_left_for_this_task=120,
+ per_run_time_limit=30,
+ resampling_strategy='cv',
+ resampling_strategy_arguments={'folds': 5},
+ )
+
+ automl.fit(self.marvin_dataset['train_X'].copy(), self.marvin_dataset['train_y'].copy())
+ automl.refit(self.marvin_dataset['train_X'].copy(), self.marvin_dataset['train_y'].copy())
+
+ # Using fitted_pipeline_ method to get model
+ self.marvin_model = automl
+
diff --git a/public-engines/iris-automl/notebooks/.ipynb_checkpoints/autosk_sample-checkpoint.ipynb b/public-engines/iris-automl/notebooks/.ipynb_checkpoints/autosk_sample-checkpoint.ipynb
new file mode 100644
index 0000000..cde2e0d
--- /dev/null
+++ b/public-engines/iris-automl/notebooks/.ipynb_checkpoints/autosk_sample-checkpoint.ipynb
@@ -0,0 +1,160 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Documentation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Sample"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "marvin_cell": "acquisitor"
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/bruno/.virtualenvs/iris-automl-engine-env/lib/python3.7/site-packages/sklearn/feature_extraction/text.py:17: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working\n",
+ " from collections import Mapping, defaultdict\n"
+ ]
+ }
+ ],
+ "source": [
+ "from sklearn import datasets\n",
+ "iris = datasets.load_iris()\n",
+ "marvin_initial_dataset = iris"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "marvin_cell": "tpreparator"
+ },
+ "outputs": [],
+ "source": [
+ "import sklearn.model_selection\n",
+ "X,y = marvin_initial_dataset.data,marvin_initial_dataset.target\n",
+ "X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, random_state=1)\n",
+ "\n",
+ "marvin_dataset = {'train_X': X_train, 'train_y': y_train, 'test_X': X_test, 'test_y': y_test}\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Cross validation example\n",
+ "\n",
+ "https://automl.github.io/auto-sklearn/master/examples/example_crossvalidation.html#sphx-glr-examples-example-crossvalidation-py"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "marvin_cell": "trainer"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[WARNING] [2019-09-11 13:10:45,749:EnsembleBuilder(1):f00fc4935fefc8f83f31eff3100a845d] No models better than random - using Dummy Score!\n",
+ "[WARNING] [2019-09-11 13:10:45,754:EnsembleBuilder(1):f00fc4935fefc8f83f31eff3100a845d] No models better than random - using Dummy Score!\n",
+ "[WARNING] [2019-09-11 13:10:47,758:EnsembleBuilder(1):f00fc4935fefc8f83f31eff3100a845d] No models better than random - using Dummy Score!\n",
+ "1\n",
+ "['/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000000.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000001.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000002.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000003.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000004.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000005.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000006.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000007.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000008.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000009.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000010.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000011.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000012.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000013.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000014.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000015.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000016.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000017.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000018.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000019.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000020.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000021.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000022.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000023.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000024.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000025.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000026.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000027.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000028.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000029.ensemble']\n"
+ ]
+ }
+ ],
+ "source": [
+ "import autosklearn.classification\n",
+ "\n",
+ "automl = autosklearn.classification.AutoSklearnClassifier(\n",
+ " time_left_for_this_task=120,\n",
+ " per_run_time_limit=30,\n",
+ " resampling_strategy='cv',\n",
+ " resampling_strategy_arguments={'folds': 5},\n",
+ ")\n",
+ "\n",
+ "automl.fit(marvin_dataset['train_X'].copy(), marvin_dataset['train_y'].copy())\n",
+ "automl.refit(marvin_dataset['train_X'].copy(), marvin_dataset['train_y'].copy())\n",
+ "\n",
+ "# Using fitted_pipeline_ method to get model\n",
+ "marvin_model = automl\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "marvin_cell": "evaluator"
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn import metrics\n",
+ "predictions = marvin_model.predict(marvin_dataset['test_X'])\n",
+ "marvin_metrics = metrics.accuracy_score(marvin_dataset['test_y'], predictions)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "marvin_cell": "ppreparator"
+ },
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "input_message = np.array([input_message])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "marvin_cell": "predictor"
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn import datasets\n",
+ "iris = datasets.load_iris()\n",
+ "prediction = marvin_model.predict(input_message)\n",
+ "final_prediction = iris.target_names[prediction][0]"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/public-engines/iris-automl/notebooks/autosk_sample.ipynb b/public-engines/iris-automl/notebooks/autosk_sample.ipynb
new file mode 100644
index 0000000..cde2e0d
--- /dev/null
+++ b/public-engines/iris-automl/notebooks/autosk_sample.ipynb
@@ -0,0 +1,160 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Documentation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Sample"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "marvin_cell": "acquisitor"
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/bruno/.virtualenvs/iris-automl-engine-env/lib/python3.7/site-packages/sklearn/feature_extraction/text.py:17: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working\n",
+ " from collections import Mapping, defaultdict\n"
+ ]
+ }
+ ],
+ "source": [
+ "from sklearn import datasets\n",
+ "iris = datasets.load_iris()\n",
+ "marvin_initial_dataset = iris"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "marvin_cell": "tpreparator"
+ },
+ "outputs": [],
+ "source": [
+ "import sklearn.model_selection\n",
+ "X,y = marvin_initial_dataset.data,marvin_initial_dataset.target\n",
+ "X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, random_state=1)\n",
+ "\n",
+ "marvin_dataset = {'train_X': X_train, 'train_y': y_train, 'test_X': X_test, 'test_y': y_test}\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Cross validation example\n",
+ "\n",
+ "https://automl.github.io/auto-sklearn/master/examples/example_crossvalidation.html#sphx-glr-examples-example-crossvalidation-py"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "marvin_cell": "trainer"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[WARNING] [2019-09-11 13:10:45,749:EnsembleBuilder(1):f00fc4935fefc8f83f31eff3100a845d] No models better than random - using Dummy Score!\n",
+ "[WARNING] [2019-09-11 13:10:45,754:EnsembleBuilder(1):f00fc4935fefc8f83f31eff3100a845d] No models better than random - using Dummy Score!\n",
+ "[WARNING] [2019-09-11 13:10:47,758:EnsembleBuilder(1):f00fc4935fefc8f83f31eff3100a845d] No models better than random - using Dummy Score!\n",
+ "1\n",
+ "['/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000000.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000001.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000002.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000003.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000004.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000005.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000006.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000007.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000008.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000009.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000010.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000011.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000012.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000013.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000014.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000015.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000016.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000017.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000018.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000019.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000020.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000021.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000022.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000023.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000024.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000025.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000026.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000027.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000028.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000029.ensemble']\n"
+ ]
+ }
+ ],
+ "source": [
+ "import autosklearn.classification\n",
+ "\n",
+ "automl = autosklearn.classification.AutoSklearnClassifier(\n",
+ " time_left_for_this_task=120,\n",
+ " per_run_time_limit=30,\n",
+ " resampling_strategy='cv',\n",
+ " resampling_strategy_arguments={'folds': 5},\n",
+ ")\n",
+ "\n",
+ "automl.fit(marvin_dataset['train_X'].copy(), marvin_dataset['train_y'].copy())\n",
+ "automl.refit(marvin_dataset['train_X'].copy(), marvin_dataset['train_y'].copy())\n",
+ "\n",
+ "# Using fitted_pipeline_ method to get model\n",
+ "marvin_model = automl\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "marvin_cell": "evaluator"
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn import metrics\n",
+ "predictions = marvin_model.predict(marvin_dataset['test_X'])\n",
+ "marvin_metrics = metrics.accuracy_score(marvin_dataset['test_y'], predictions)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "marvin_cell": "ppreparator"
+ },
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "input_message = np.array([input_message])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "marvin_cell": "predictor"
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn import datasets\n",
+ "iris = datasets.load_iris()\n",
+ "prediction = marvin_model.predict(input_message)\n",
+ "final_prediction = iris.target_names[prediction][0]"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/public-engines/iris-automl/notebooks/tpot_sample.ipynb b/public-engines/iris-automl/notebooks/tpot_sample.ipynb
new file mode 100644
index 0000000..2cfba15
--- /dev/null
+++ b/public-engines/iris-automl/notebooks/tpot_sample.ipynb
@@ -0,0 +1,179 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Documentation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Sample"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sklearn import datasets\n",
+ "iris = datasets.load_iris()\n",
+ "marvin_initial_dataset = iris"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import sklearn.model_selection\n",
+ "X,y = marvin_initial_dataset.data,marvin_initial_dataset.target\n",
+ "X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, random_state=1)\n",
+ "\n",
+ "marvin_dataset = {'train_X': X_train, 'train_y': y_train, 'test_X': X_test, 'test_y': y_test}\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Warning: xgboost.XGBClassifier is not available and will not be used by TPOT.\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(IntProgress(value=0, description='Optimization Progress', max=120, style=ProgressStyle(descript…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Generation 1 - Current best internal CV score: 0.982213438735178\n",
+ "Generation 2 - Current best internal CV score: 0.982213438735178\n",
+ "Generation 3 - Current best internal CV score: 0.982213438735178\n",
+ "Generation 4 - Current best internal CV score: 0.982213438735178\n",
+ "Generation 5 - Current best internal CV score: 0.982213438735178\n",
+ "\n",
+ "Best pipeline: GaussianNB(Normalizer(input_matrix, norm=l2))\n"
+ ]
+ }
+ ],
+ "source": [
+ "from tpot import TPOTClassifier\n",
+ "\n",
+ "pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n",
+ " random_state=42, verbosity=2)\n",
+ "\n",
+ "pipeline_optimizer.fit(marvin_dataset['train_X'].copy(), marvin_dataset['train_y'].copy())\n",
+ "\n",
+ "# Using fitted_pipeline_ method to get model\n",
+ "marvin_model = pipeline_optimizer.fitted_pipeline_\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sklearn import metrics\n",
+ "predictions = marvin_model.predict(marvin_dataset['test_X'])\n",
+ "marvin_metrics = metrics.accuracy_score(marvin_dataset['test_y'], predictions)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "NameError",
+ "evalue": "name 'input_message' is not defined",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
+ "\u001b[0;32m<ipython-input-9-e2c2e30e02dc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0minput_message\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minput_message\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+ "\u001b[0;31mNameError\u001b[0m: name 'input_message' is not defined"
+ ]
+ }
+ ],
+ "source": [
+ "import numpy as np\n",
+ "input_message = np.array([input_message])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "NameError",
+ "evalue": "name 'input_message' is not defined",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
+ "\u001b[0;32m<ipython-input-10-7c6c94f5195d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdatasets\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0miris\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatasets\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_iris\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mprediction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmarvin_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_message\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mfinal_prediction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0miris\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtarget_names\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mprediction\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;31mNameError\u001b[0m: name 'input_message' is not defined"
+ ]
+ }
+ ],
+ "source": [
+ "from sklearn import datasets\n",
+ "iris = datasets.load_iris()\n",
+ "prediction = marvin_model.predict(input_message)\n",
+ "final_prediction = iris.target_names[prediction][0]\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/public-engines/iris-automl/pytest.ini b/public-engines/iris-automl/pytest.ini
new file mode 100644
index 0000000..cab8644
--- /dev/null
+++ b/public-engines/iris-automl/pytest.ini
@@ -0,0 +1,4 @@
+[pytest]
+minversion = 2.0
+norecursedirs = .git .tox .eggs .cache *.egg build dist tmp*
+python_files = test*.py
\ No newline at end of file
diff --git a/public-engines/iris-automl/requirements.txt b/public-engines/iris-automl/requirements.txt
new file mode 100644
index 0000000..983d39a
--- /dev/null
+++ b/public-engines/iris-automl/requirements.txt
@@ -0,0 +1,20 @@
+setuptools
+nose
+Cython
+
+numpy>=1.9.0
+scipy>=0.14.1
+
+scikit-learn>=0.21.0,<0.22
+
+lockfile
+joblib
+psutil
+pyyaml
+liac-arff
+pandas
+
+ConfigSpace>=0.4.0,<0.5
+pynisher>=0.4.2
+pyrfr>=0.7,<0.9
+smac==0.8
diff --git a/public-engines/iris-automl/setup.cfg b/public-engines/iris-automl/setup.cfg
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/public-engines/iris-automl/setup.cfg
diff --git a/public-engines/iris-automl/setup.py b/public-engines/iris-automl/setup.py
new file mode 100644
index 0000000..afe1d13
--- /dev/null
+++ b/public-engines/iris-automl/setup.py
@@ -0,0 +1,175 @@
+from __future__ import print_function
+
+import os
+import shutil
+from os.path import dirname, join
+from setuptools import setup, find_packages
+from setuptools.command.test import test as TestCommand
+from setuptools.command.develop import develop as _develop
+from setuptools.command.install import install as _install
+
+
+REQUIREMENTS_TESTS = [
+ 'pytest>=2.6.4',
+ 'pytest-cov>=1.8.1',
+ 'mock>=2.0.0',
+ 'virtualenv>=15.0.1',
+ 'tox>=2.2.0',
+]
+
+def _get_version():
+ """Return the project version from VERSION file."""
+
+ with open(join(dirname(__file__), 'marvin_iris_automl/VERSION'), 'rb') as f:
+ version = f.read().decode('ascii').strip()
+ return version
+
+
+def _hooks(dir):
+ _set_autocomplete()
+ _install_notebook_extension()
+
+
+def _set_autocomplete():
+ import marvin_python_toolbox as toolbox
+ virtualenv = os.environ.get('VIRTUAL_ENV', None)
+
+ if virtualenv:
+ postactivate = os.path.join(virtualenv, 'bin', 'postactivate')
+
+ if os.path.exists(postactivate):
+ shutil.copy(
+ os.path.join(toolbox.__path__[0], 'extras', 'marvin_bash_completion'),
+ os.path.join(virtualenv, 'marvin_bash_completion')
+ )
+
+ command = 'source "{}"'.format(os.path.join(virtualenv, 'marvin_bash_completion'))
+
+ with open(postactivate, 'r+') as fp:
+ lines = fp.readlines()
+ fp.seek(0)
+ configured = False
+ for line in lines:
+ if 'marvin_bash_completion' in line:
+ # Replacing old autocomplete configuration
+ fp.write(command)
+ configured = True
+ else:
+ fp.write(line)
+
+ if not configured:
+ fp.write(command)
+ # 'Autocomplete was successfully configured'
+ fp.write('\n')
+ fp.truncate()
+
+
+def _install_notebook_extension():
+ import marvin_python_toolbox as toolbox
+
+ install_command = [
+ "jupyter",
+ "nbextension",
+ "install",
+ os.path.join(toolbox.__path__[0], 'extras', 'notebook_extensions', 'main.js'),
+ "--destination",
+ "marvin.js",
+ "--sys-prefix",
+ "--overwrite"
+ ]
+
+ os.system(' '.join(install_command))
+
+ enable_command = [
+ "jupyter",
+ "nbextension",
+ "enable",
+ "marvin",
+ "--sys-prefix"
+ ]
+
+ os.system(' '.join(enable_command))
+
+
+class develop(_develop):
+ def run(self):
+ _develop.run(self)
+ self.execute(_hooks, (self.install_lib,), msg="Running develop preparation task")
+
+
+class install(_install):
+ def run(self):
+ _install.run(self)
+ self.execute(_hooks, (self.install_lib,), msg="Running install preparation task")
+
+
+class Tox(TestCommand):
+ """Run the test cases using TOX command."""
+
+ user_options = [('tox-args=', 'a', "Arguments to pass to tox")]
+
+ def initialize_options(self):
+ TestCommand.initialize_options(self)
+ self.tox_args = None
+
+ def finalize_options(self):
+ TestCommand.finalize_options(self)
+ self.test_args = []
+ self.test_suite = True
+
+ def run_tests(self):
+ # Import here, cause outside the eggs aren't loaded
+ import tox
+ import shlex
+ import sys
+ args = self.tox_args
+ if args:
+ args = shlex.split(self.tox_args)
+ else:
+ # Run all tests by default
+ args = ['-c', join(dirname(__file__), 'tox.ini'), 'tests']
+ errno = tox.cmdline(args=args)
+ sys.exit(errno)
+
+
+setup(
+ name='marvin_iris_automl',
+ version=_get_version(),
+ url='',
+ description='Marvin engine',
+ long_description=open(join(dirname(__file__), 'README.md')).read(),
+ author='Marvin AI Community',
+ maintainer='Marvin AI Community',
+ maintainer_email='dev@marvin.apache.org',
+ packages=find_packages(exclude=('tests', 'tests.*')),
+ include_package_data=True,
+ zip_safe=False,
+ classifiers=[
+ 'Development Status :: 3 - Alpha',
+ 'Intended Audience :: Developers',
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.3',
+ 'Programming Language :: Python :: 3.4',
+ 'Programming Language :: Python :: 3.5',
+ 'Topic :: Software Development :: Libraries :: Python Modules',
+ ],
+ install_requires=[
+ 'scikit-learn>=0.18.2',
+ 'scipy>=0.19.1',
+ 'numpy>=1.13.1',
+ 'pandas>=0.20.3',
+ 'matplotlib>=2.0.2',
+ 'marvin-python-toolbox>=0.0.5',
+ 'Fabric>=1.14.0',
+ ],
+ tests_require=REQUIREMENTS_TESTS,
+ extras_require={
+ 'testing': REQUIREMENTS_TESTS,
+ },
+ cmdclass={
+ 'test': Tox, 'develop': develop, 'install': install
+ },
+)
\ No newline at end of file
diff --git a/public-engines/iris-automl/tests/conftest.py b/public-engines/iris-automl/tests/conftest.py
new file mode 100644
index 0000000..efd37f3
--- /dev/null
+++ b/public-engines/iris-automl/tests/conftest.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os
+import pytest
+
+os.environ['TESTING'] = 'True'
+
+
+@pytest.fixture
+def mocked_params():
+ return {'params': 1}
diff --git a/public-engines/iris-automl/tests/data_handler/test_acquisitor_and_cleaner.py b/public-engines/iris-automl/tests/data_handler/test_acquisitor_and_cleaner.py
new file mode 100644
index 0000000..fcb794d
--- /dev/null
+++ b/public-engines/iris-automl/tests/data_handler/test_acquisitor_and_cleaner.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_automl.data_handler import AcquisitorAndCleaner
+
+
+class TestAcquisitorAndCleaner:
+ def test_execute(self, mocked_params):
+ ac = AcquisitorAndCleaner()
+ ac.execute(params=mocked_params)
+ assert not ac._params
diff --git a/public-engines/iris-automl/tests/data_handler/test_training_preparator.py b/public-engines/iris-automl/tests/data_handler/test_training_preparator.py
new file mode 100644
index 0000000..d151766
--- /dev/null
+++ b/public-engines/iris-automl/tests/data_handler/test_training_preparator.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_automl.data_handler import TrainingPreparator
+
+
+class TestTrainingPreparator:
+ def test_execute(self, mocked_params):
+ ac = TrainingPreparator()
+ ac.execute(params=mocked_params)
+ assert not ac._params
\ No newline at end of file
diff --git a/public-engines/iris-automl/tests/prediction/test_feedback.py b/public-engines/iris-automl/tests/prediction/test_feedback.py
new file mode 100644
index 0000000..3354b7b
--- /dev/null
+++ b/public-engines/iris-automl/tests/prediction/test_feedback.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_automl.prediction import Feedback
+
+
+class TestFeedback:
+ def test_execute(self, mocked_params):
+ fb = Feedback()
+ fb.execute(input_message="fake message", params=mocked_params)
+ assert not fb._params
\ No newline at end of file
diff --git a/public-engines/iris-automl/tests/prediction/test_prediction_preparator.py b/public-engines/iris-automl/tests/prediction/test_prediction_preparator.py
new file mode 100644
index 0000000..4006cd7
--- /dev/null
+++ b/public-engines/iris-automl/tests/prediction/test_prediction_preparator.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_automl.prediction import PredictionPreparator
+
+
+class TestPredictionPreparator:
+ def test_execute(self, mocked_params):
+ ac = PredictionPreparator()
+ ac.execute(input_message="fake message", params=mocked_params)
+ assert not ac._params
\ No newline at end of file
diff --git a/public-engines/iris-automl/tests/prediction/test_predictor.py b/public-engines/iris-automl/tests/prediction/test_predictor.py
new file mode 100644
index 0000000..46fddf7
--- /dev/null
+++ b/public-engines/iris-automl/tests/prediction/test_predictor.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_automl.prediction import Predictor
+
+
+class TestPredictor:
+ def test_execute(self, mocked_params):
+ ac = Predictor()
+ ac.execute(input_message="fake message", params=mocked_params)
+ assert not ac._params
\ No newline at end of file
diff --git a/public-engines/iris-automl/tests/training/test_metrics_evaluator.py b/public-engines/iris-automl/tests/training/test_metrics_evaluator.py
new file mode 100644
index 0000000..376bbea
--- /dev/null
+++ b/public-engines/iris-automl/tests/training/test_metrics_evaluator.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_automl.training import MetricsEvaluator
+
+
+class TestMetricsEvaluator:
+ def test_execute(self, mocked_params):
+ ac = MetricsEvaluator()
+ ac.execute(params=mocked_params)
+ assert not ac._params
\ No newline at end of file
diff --git a/public-engines/iris-automl/tests/training/test_trainer.py b/public-engines/iris-automl/tests/training/test_trainer.py
new file mode 100644
index 0000000..49ba506
--- /dev/null
+++ b/public-engines/iris-automl/tests/training/test_trainer.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_automl.training import Trainer
+
+
+class TestTrainer:
+ def test_execute(self, mocked_params):
+ ac = Trainer()
+ ac.execute(params=mocked_params)
+ assert not ac._params
diff --git a/public-engines/iris-automl/tox.ini b/public-engines/iris-automl/tox.ini
new file mode 100644
index 0000000..2019932
--- /dev/null
+++ b/public-engines/iris-automl/tox.ini
@@ -0,0 +1,8 @@
+[tox]
+envlist = py27
+
+[testenv]
+deps=pytest
+ pytest-cov
+ mock
+commands=py.test --cov={envsitepackagesdir}/marvin_iris_automl --cov-report html --cov-report xml {posargs}
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/.bumpversion.cfg b/public-engines/iris-h2o-automl/.bumpversion.cfg
new file mode 100644
index 0000000..c1f036e
--- /dev/null
+++ b/public-engines/iris-h2o-automl/.bumpversion.cfg
@@ -0,0 +1,5 @@
+[bumpversion]
+current_version = 0.0.1
+
+[bumpversion:file:marvin_iris_h2o_automl/VERSION]
+[bumpversion:file:README.md]
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/.coveragerc b/public-engines/iris-h2o-automl/.coveragerc
new file mode 100644
index 0000000..6ebe78e
--- /dev/null
+++ b/public-engines/iris-h2o-automl/.coveragerc
@@ -0,0 +1,22 @@
+[run]
+omit = tests/*
+branch = True
+
+[report]
+exclude_lines =
+ pragma: no cover
+
+ def __repr__
+ if self\.debug
+
+ raise AssertionError
+ raise NotImplementedError
+
+ if 0:
+ if __name__ == .__main__.:
+
+[html]
+directory = coverage_report
+
+[xml]
+output = coverage_report.xml
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/.gitignore b/public-engines/iris-h2o-automl/.gitignore
new file mode 100644
index 0000000..05dd2a2
--- /dev/null
+++ b/public-engines/iris-h2o-automl/.gitignore
@@ -0,0 +1,16 @@
+.cache
+.eggs
+.tox
+.testmondata
+.coverage
+.coverage.*
+coverage_report.xml
+coverage_report
+*.egg
+*.egg-info
+*.pyc
+tests/__pycache__
+.DS_Store
+.packages
+.profiling
+notebooks/data
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/CHANGES.md b/public-engines/iris-h2o-automl/CHANGES.md
new file mode 100644
index 0000000..4835f6f
--- /dev/null
+++ b/public-engines/iris-h2o-automl/CHANGES.md
@@ -0,0 +1,5 @@
+## Changes log
+
+### 0.0.1
+
+ - initial version
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/Dockerfile b/public-engines/iris-h2o-automl/Dockerfile
new file mode 100644
index 0000000..f117241
--- /dev/null
+++ b/public-engines/iris-h2o-automl/Dockerfile
@@ -0,0 +1,113 @@
+############################################################
+FROM python:2-alpine3.10
+# To use python3 comment the line above and uncomment
+# the line bellow.
+#FROM python:3-alpine3.10
+############################################################
+
+MAINTAINER dev@marvin.apache.org
+
+ENV SLEEP_MILLIS 0
+
+USER root
+
+##############################################################
+# Define all environment variables to be used
+##############################################################
+
+ENV MARVIN_HOME=/opt/marvin
+ENV MARVIN_DATA_PATH=/marvin-data
+ENV MARVIN_ENGINE_HOME=$MARVIN_HOME/engine
+ENV MARVIN_ENGINE_ENV=marvin-engine-env
+ENV WORKON_HOME=$MARVIN_HOME/.virtualenvs
+ENV SPARK_HOME=/opt/spark
+ENV SPARK_CONF_DIR=$SPARK_HOME/conf
+ENV HADOOP_CONF_DIR=$SPARK_CONF_DIR
+ENV YARN_CONF_DIR=$SPARK_CONF_DIR
+
+
+
+##############################################################
+# Create all folders needed
+##############################################################
+
+RUN mkdir -p $MARVIN_HOME && \
+ mkdir -p $MARVIN_DATA_PATH && \
+ mkdir -p $MARVIN_ENGINE_HOME && \
+ mkdir -p /var/log/marvin/engines && \
+ mkdir -p /var/run/marvin/engines && \
+##############################################################
+# Install the system dependencies for default installation
+##############################################################
+ apk add --no-cache g++ openssl-dev openjdk11-jre-headless bash && \
+ apk add --no-cache --virtual .build-deps make \
+ git \
+ wget \
+ libsass-dev \
+ openblas-dev \
+ libffi-dev \
+ libxml2-dev \
+ libxslt-dev \
+ libpng-dev \
+ freetype-dev \
+ cyrus-sasl-dev
+##############################################################
+# Install Apache Spark
+#
+# Uncomment if you are using spark, note that is needed the
+# spark configuration files to the think works correctly.
+##############################################################
+#
+# RUN wget -O /tmp/spark-2.1.1-bin-hadoop2.6.tgz https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz && \
+# tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/ && \
+# ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark
+##############################################################
+
+RUN mkdir -p $SPARK_CONF_DIR
+
+##############################################################
+# <CUSTOM ENGINE INSTALLATION PROCEDURE HERE> #
+##############################################################
+
+
+##############################################################
+# Copy and Install the marvin engine
+##############################################################
+
+RUN /bin/bash -c "pip install virtualenvwrapper && \
+ cd $MARVIN_ENGINE_HOME && \
+ source /usr/local/bin/virtualenvwrapper.sh && \
+ mkvirtualenv $MARVIN_ENGINE_ENV"
+
+ADD build/engine.tar $MARVIN_ENGINE_HOME
+
+ADD build/marvin-engine-executor-assembly.jar $MARVIN_DATA_PATH
+
+RUN /bin/bash -c "source /usr/local/bin/virtualenvwrapper.sh && \
+ workon $MARVIN_ENGINE_ENV && \
+ cd $MARVIN_ENGINE_HOME && \
+ pip install --no-cache numpy && \
+ pip install --no-cache scipy && \
+ pip install --no-cache pandas && \
+ pip install --no-cache matplotlib && \
+ pip install --no-cache cython && \
+ pip install --no-cache scikit-learn && \
+ pip install --no-cache Fabric && \
+ pip install --no-cache marvin-python-toolbox && \
+ pip install . "
+##############################################################
+# Uninstalling unnecessary software and cleaning cache
+##############################################################
+RUN rm -rf /root/.cache && \
+ apk del .build-deps
+
+##############################################################
+# Starts the engine http server
+##############################################################
+
+EXPOSE 8000
+
+CMD /bin/bash -c "source /usr/local/bin/virtualenvwrapper.sh && \
+ workon $MARVIN_ENGINE_ENV && \
+ cd $MARVIN_ENGINE_HOME && \
+ marvin engine-httpserver -h 0.0.0.0 -p 8000"
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/INSTALL b/public-engines/iris-h2o-automl/INSTALL
new file mode 100644
index 0000000..fccdaf8
--- /dev/null
+++ b/public-engines/iris-h2o-automl/INSTALL
@@ -0,0 +1 @@
+REPLACE: Add here the detailed instructions to install this project
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/LICENSE b/public-engines/iris-h2o-automl/LICENSE
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/public-engines/iris-h2o-automl/LICENSE
diff --git a/public-engines/iris-h2o-automl/MANIFEST.in b/public-engines/iris-h2o-automl/MANIFEST.in
new file mode 100644
index 0000000..5ab120a
--- /dev/null
+++ b/public-engines/iris-h2o-automl/MANIFEST.in
@@ -0,0 +1,9 @@
+include CHANGES.md
+include INSTALL
+include LICENSE
+include MANIFEST.in
+include README.md
+include marvin_iris_h2o_automl/VERSION
+recursive-include notebooks *
+prune notebooks/build
+recursive-include tests *
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/Makefile b/public-engines/iris-h2o-automl/Makefile
new file mode 100644
index 0000000..925293f
--- /dev/null
+++ b/public-engines/iris-h2o-automl/Makefile
@@ -0,0 +1,81 @@
+.PHONY: help marvin marvin-automl marvin-prod update clean-pyc clean-build clean-reports clean-deps clean docker-build docker-push docker-run
+
+DOCKER_VERSION?=0.00.01
+DOCKER_REGISTRY_ADRESS?=docker.registry.io
+MARVIN_DATA_PATH?=$(HOME)/marvin/data
+MARVIN_ENGINE_NAME?=iris_h2o_automl
+MARVIN_TOOLBOX_VERSION?=0.0.5
+
+help:
+ @echo " marvin"
+ @echo " Prepare project to be used as a marvin package."
+ @echo " marvin-prod"
+ @echo " Prepare project to be used in production environment."
+ @echo " marvin-automl"
+ @echo " Prepare project to be used as a marvin-automl package."
+ @echo " update"
+ @echo " Reinstall requirements and setup.py dependencies."
+ @echo " clean"
+ @echo " Remove all generated artifacts."
+ @echo " clean-pyc"
+ @echo " Remove python artifacts."
+ @echo " clean-build"
+ @echo " Remove build artifacts."
+ @echo " clean-reports"
+ @echo " Remove coverage reports."
+ @echo " clean-deps"
+ @echo " Remove marvin setup.py dependencies."
+ @echo " docker-build"
+ @echo " Runs the docker build command with marvin env default parameters."
+ @echo " docker-push"
+ @echo " Runs the docker push command with marvin env default parameters."
+ @echo " docker-run"
+ @echo " Runs the docker run command with marvin env default parameters."
+
+marvin:
+ pip install -e ".[testing]"
+ marvin --help
+marvin-automl:
+ pip install -e ".[testing]"
+ bash install_automl.sh
+ marvin --help
+marvin-prod:
+ pip install .
+ marvin --help
+
+update:
+ pip install -e . -U
+
+clean-pyc:
+ find . -name '*.pyc' -exec rm -f {} +
+ find . -name '*.pyo' -exec rm -f {} +
+ find . -name '*~' -exec rm -f {} +
+
+clean-build:
+ rm -rf *.egg-info
+ rm -rf .cache
+ rm -rf .eggs
+ rm -rf dist
+ rm -rf build
+
+clean-reports:
+ rm -rf coverage_report/
+ rm -f coverage.xml
+ rm -f .coverage
+
+clean-deps:
+ pip freeze | grep -v "^-e" | xargs pip uninstall -y
+
+clean: clean-build clean-pyc clean-reports clean-deps
+
+docker-build: clean-build
+ mkdir -p build
+ tar -cf build/engine.tar --exclude=*.log --exclude=*.pkl --exclude='build' --exclude='notebooks' --exclude=*.tar *
+ cp -f $(MARVIN_DATA_PATH)/marvin-engine-executor-assembly-$(MARVIN_TOOLBOX_VERSION).jar build/marvin-engine-executor-assembly.jar
+ sudo docker build -t $(DOCKER_REGISTRY_ADRESS)/$(MARVIN_ENGINE_NAME):$(DOCKER_VERSION) .
+
+docker-run:
+ sudo docker run --name=marvin-$(MARVIN_ENGINE_NAME)-$(DOCKER_VERSION) --mount type=bind,source=$(MARVIN_DATA_PATH),destination=/marvin-data -p 8000:8000 $(DOCKER_REGISTRY_ADRESS)/$(MARVIN_ENGINE_NAME):$(DOCKER_VERSION)
+
+docker-push:
+ sudo docker push $(DOCKER_REGISTRY_ADRESS)/$(MARVIN_ENGINE_NAME):$(DOCKER_VERSION)
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/README.md b/public-engines/iris-h2o-automl/README.md
new file mode 100644
index 0000000..6308f72
--- /dev/null
+++ b/public-engines/iris-h2o-automl/README.md
@@ -0,0 +1,142 @@
+# iris_h2o_automl v0.0.1
+
+## Overview
+
+Iris classifier with H2O AutoML using a custom serializer.
+
+
+## Requirements
+
+ - Python 2.7
+ - Numpy 1.11.0 or higher
+ - H2O (automaticaly installed with Marvin AutoML script)
+
+
+## Installation
+
+Use the Marvin toolbox to provision, deploy and start the remote HTTP server.
+
+First, edit the `marvin.ini` file, setting the options within the
+`ssh_deployment` section:
+
+1. `host`: the host IP address or name where the engine should be deployed. You
+can enable multi-host deployment using `,` to separate hosts
+2. `port`: the SSH connection port
+3. `user`: the SSH connection username. Currently, only a single user is
+supported. This user should be capable of *passwordless sudo*, although it can
+use password for the SSH connection
+
+Next, ensure that the remotes servers are provisioned (all required software
+are installed):
+
+ marvin engine-deploy --provision
+
+Next, package your engine:
+
+ marvin engine-deploy --package
+
+This will create a compressed archive containing your engine code under the
+`.packages` directory.
+
+Next, deploy your engine to remotes servers:
+
+ marvin engine-deploy
+
+By default, a dependency clean will be executed at each deploy. You can skip it
+using:
+
+ marvin engine-deploy --skip-clean
+
+Next, you can start the HTTP server in the remotes servers:
+
+ marvin engine-httpserver-remote start
+
+You can check if the HTTP server is running:
+
+ marvin engine-httpserver-remote status
+
+And stop it:
+
+ marvin engine-httpserver-remote stop
+
+After starting, you can test it by making a HTTP request to any endpoint, like:
+
+ curl -v http://example.com/predictor/health
+
+Under the hood, this engine uses Fabric to define provisioning and deployment
+process. Check the `fabfile.py` for more information. You can add new tasks or
+edit existing ones to match your provisioning and deployment pipeline.
+
+## Development
+
+### Getting started
+
+First, create a new virtualenv
+
+```
+mkvirtualenv marvin_iris_h2o_automl_env
+```
+
+Now install the development dependencies
+
+```
+make marvin
+```
+
+You are now ready to code.
+
+
+### Adding new dependencies
+
+It\`s very important. All development dependencies should be added to `setup.py`.
+
+### Running tests
+
+This project uses *[py.test](http://pytest.org/)* as test runner and *[Tox](https://tox.readthedocs.io)* to manage virtualenvs.
+
+To run all tests use the following command
+
+```
+marvin test
+```
+
+To run specific test
+
+```
+marvin test tests/test_file.py::TestClass::test_method
+```
+
+
+### Writting documentation
+
+The project documentation is written using *[Jupyter](http://jupyter.readthedocs.io/)* notebooks.
+You can start the notebook server from the command line by running the following command
+
+```
+marvin notebook
+```
+
+Use notebooks to demonstrate how to use the lib features. It can also be useful to show some use cases.
+
+
+### Bumping version
+
+```
+marvin pkg-bumpversion [patch|minor|major]
+git add . && git commit -m "Bump version"
+```
+
+
+### Tagging version
+
+```
+marvin pkg-createtag
+git push origin master --follow-tags
+```
+
+
+### Logging
+
+The default log level is set to _WARNING_. You can change the log level at runtime setting another value to one of the following environment variable: `MARVIN_IRIS_H2O_AUTOML_LOG_LEVEL` or `LOG_LEVEL`. The available values are _CRITICAL_, _ERROR_, _WARNING_, _INFO_ and _DEBUG_.
+
+Be careful using `LOG_LEVEL`, it may affect another lib.
diff --git a/public-engines/iris-h2o-automl/docs.yaml b/public-engines/iris-h2o-automl/docs.yaml
new file mode 100644
index 0000000..4255e96
--- /dev/null
+++ b/public-engines/iris-h2o-automl/docs.yaml
@@ -0,0 +1,567 @@
+openapi: "3.0.0"
+info:
+ version: 0.0.1
+ title: marvin_iris_h2o_automl API Doc
+ contact:
+ name: mantainer Marvin AI Community
+ email: dev@marvin.apache.org
+ url: https://marvin.apache.org
+ license:
+ name: Apache License 2.0
+servers:
+ - url: http://localhost:8000
+ - url: http://0.0.0.0:8000
+tags:
+ - name: Docker
+ description: For Docker users, please use "make docker-build" and "make docker-run" commands in your engine virtualenv to start the server
+ - name: Acquisitor
+ description: Setup the initial_dataset with all cleaned data necessary to build your dataset in the next action
+ - name: Tpreparator
+ description: Setup the dataset with the transformed data that is compatible with the algorithm used to build the model in the next action
+ - name: Trainer
+ description: Setup the model with the result of the algorithm used to training
+ - name: Evaluator
+ description: Setup the metrics with the result of the algorithms used to test the model
+ - name: Predictor
+ description: Return the predicted value in a json parsable object format
+ - name: Feedback
+ description: Receive feedback message, user can manipulate this message for any use
+ - name: Pipeline
+ description: Perform all batch actions in your right order
+paths:
+ /acquisitor/health:
+ get:
+ summary: Get acquisitor's service health
+ operationId: getAcquisitorHealth
+ tags:
+ - Acquisitor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /acquisitor/status:
+ get:
+ summary: Get acquisitor's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getAcquisitorStatus
+ tags:
+ - Acquisitor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /acquisitor:
+ post:
+ summary: Run acquisitor
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: acquisitor
+ tags:
+ - Acquisitor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator/health:
+ get:
+ summary: Get trainer preparator's service health
+ operationId: getTPreparatortorHealth
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator/status:
+ get:
+ summary: Get trainer preparator's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getTPreparatorStatus
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator/reload?protocol=:
+ put:
+ summary: Reload artifact for trainer preparator
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadTPreparator
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator:
+ post:
+ summary: Run trainer preparator
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: tpreparator
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer/health:
+ get:
+ summary: Get trainer's service health
+ operationId: getTrainerHealth
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer/status:
+ get:
+ summary: Get trainer's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getTrainerStatus
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer/reload?protocol=:
+ put:
+ summary: Reload artifact for trainer
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadTrainer
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer:
+ post:
+ summary: Run trainer
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: trainer
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/health:
+ get:
+ summary: Get evaluator's service health
+ operationId: getEvaluatorHealth
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/status:
+ get:
+ summary: Get evaluator's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getEvaluatorStatus
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/metrics?protocol=:
+ get:
+ summary: Get metrics's value
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: Metrics protocol value
+ operationId: getMetrics
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/reload?protocol=:
+ put:
+ summary: Reload artifact for evaluator
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: reloadEvaluator
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator:
+ post:
+ summary: Run evaluator
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: evaluator
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor/health:
+ get:
+ summary: Get predictor's service health
+ operationId: getPredictorHealth
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor/status:
+ get:
+ summary: Get predictor's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getPredictorStatus
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor/reload?protocol=:
+ put:
+ summary: Reload artifact for predictor
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadPredictor
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor:
+ post:
+ summary: Run predictor
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: predictor
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback/health:
+ get:
+ summary: Get feedback's service health
+ operationId: getFeedbackHealth
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback/status:
+ get:
+ summary: Get feedback's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getFeedbackStatus
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback/reload?protocol=:
+ put:
+ summary: Reload artifact for feedback
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadFeedback
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback:
+ post:
+ summary: Run feedback
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: feedback
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /pipeline:
+ post:
+ summary: Do all batch actions (from Acquisitor to Evaluator)
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: pipeline
+ tags:
+ - Pipeline
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/engine.messages b/public-engines/iris-h2o-automl/engine.messages
new file mode 100644
index 0000000..3d8c0f5
--- /dev/null
+++ b/public-engines/iris-h2o-automl/engine.messages
@@ -0,0 +1 @@
+[[1,2,3,4]]
diff --git a/public-engines/iris-h2o-automl/engine.metadata b/public-engines/iris-h2o-automl/engine.metadata
new file mode 100644
index 0000000..78f9df2
--- /dev/null
+++ b/public-engines/iris-h2o-automl/engine.metadata
@@ -0,0 +1,62 @@
+{
+ "name": "iris_h2o_automl",
+ "version": "v0.0.1",
+ "engineType": "python",
+ "artifactsRemotePath": "/tmp/marvin",
+ "artifactManagerType": "FS",
+ "onlineActionTimeout": 1000,
+ "metricsTimeout": 10000,
+ "healthCheckTimeout": 2000,
+ "reloadTimeout": 600000,
+ "batchActionTimeout": 600000,
+ "pipelineActions": ["acquisitor", "tpreparator", "trainer", "evaluator"],
+ "actions": [{
+ "name": "acquisitor",
+ "actionType": "batch",
+ "port": 50051,
+ "host": "localhost",
+ "artifactsToPersist": ["initialdataset"],
+ "artifactsToLoad": [],
+ "pipeline": []
+ }, {
+ "name": "tpreparator",
+ "actionType": "batch",
+ "port": 50052,
+ "host": "localhost",
+ "artifactsToPersist": ["dataset"],
+ "artifactsToLoad": ["initialdataset"],
+ "pipeline": []
+ }, {
+ "name": "trainer",
+ "actionType": "batch",
+ "port": 50053,
+ "host": "localhost",
+ "artifactsToPersist": ["model"],
+ "artifactsToLoad": ["dataset"],
+ "pipeline": []
+ }, {
+ "name": "evaluator",
+ "actionType": "batch",
+ "port": 50054,
+ "host": "localhost",
+ "artifactsToPersist": ["metrics"],
+ "artifactsToLoad": ["dataset", "model"],
+ "pipeline": []
+ }, {
+ "name": "predictor",
+ "actionType": "online",
+ "port": 50055,
+ "host": "localhost",
+ "artifactsToPersist": [],
+ "artifactsToLoad": ["model", "metrics"],
+ "pipeline": ["ppreparator"]
+ }, {
+ "name": "feedback",
+ "actionType": "online",
+ "port": 50056,
+ "host": "localhost",
+ "artifactsToPersist": [],
+ "artifactsToLoad": [],
+ "pipeline": []
+ }]
+}
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/engine.params b/public-engines/iris-h2o-automl/engine.params
new file mode 100644
index 0000000..410f586
--- /dev/null
+++ b/public-engines/iris-h2o-automl/engine.params
@@ -0,0 +1,3 @@
+{
+ "PARAM_1" : "VALUE_OF_PARAM_1"
+}
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/fabfile.py b/public-engines/iris-h2o-automl/fabfile.py
new file mode 100644
index 0000000..a474f9b
--- /dev/null
+++ b/public-engines/iris-h2o-automl/fabfile.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+from fabric.api import env
+from fabric.api import run
+from fabric.api import execute
+from fabric.api import cd
+from fabric.api import local
+from fabric.api import put
+from fabric.api import sudo
+from fabric.state import output
+from marvin_python_toolbox import __version__ as TOOLBOX_VERSION
+from marvin_python_toolbox.common.config import Config
+
+_host = Config.get("host", section="ssh_deployment").split(",")
+_port = Config.get("port", section="ssh_deployment")
+_user = Config.get("user", section="ssh_deployment")
+
+for h in _host:
+ env.hosts.append("{user}@{host}:{port}".format(user=_user, host=h, port=_port))
+
+output["everything"] = False
+output["running"] = True
+
+env.package = "marvin_iris_h2o_automl"
+env.margin_engine_executor_prefix = "/opt/marvin/engine-executor"
+env.margin_engine_executor_jar = "marvin-engine-executor-assembly-{version}.jar".format(version=TOOLBOX_VERSION)
+env.marvin_engine_executor_path = env.margin_engine_executor_prefix + "/" + env.margin_engine_executor_jar
+
+
+def install_oracle_jdk():
+ sudo("add-apt-repository ppa:webupd8team/java -y")
+ sudo("apt-get -qq update")
+ run("echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections")
+ run("echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections")
+ sudo("apt-get install -y oracle-java8-installer")
+
+
+def install_virtualenvwrapper():
+ run("pip install virtualenvwrapper")
+ run("echo 'export WORKON_HOME=${HOME}/.virtualenvs' >> ${HOME}/.profile")
+ run("echo 'source /usr/local/bin/virtualenvwrapper.sh' >> ${HOME}/.profile")
+
+
+def install_apache_spark():
+ run("curl https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz -o /tmp/spark-2.1.1-bin-hadoop2.6.tgz")
+ sudo("tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/")
+ sudo("ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark")
+ run("echo 'export SPARK_HOME=/opt/spark' >> ${HOME}/.profile")
+
+
+def install_required_packages():
+ sudo("apt-get update -y")
+ sudo("apt-get install -y git")
+ sudo("apt-get install -y wget")
+ sudo("apt-get install -y python2.7-dev")
+ sudo("apt-get install -y python-pip")
+ sudo("apt-get install -y ipython")
+ sudo("apt-get install -y libffi-dev")
+ sudo("apt-get install -y libssl-dev")
+ sudo("apt-get install -y libxml2-dev")
+ sudo("apt-get install -y libxslt1-dev")
+ sudo("apt-get install -y libpng12-dev")
+ sudo("apt-get install -y libfreetype6-dev")
+ sudo("apt-get install -y python-tk")
+ sudo("apt-get install -y libsasl2-dev")
+ sudo("apt-get install -y python-pip")
+ sudo("apt-get install -y graphviz")
+ sudo("pip install --upgrade pip")
+
+
+def install_marvin_engine_executor():
+ sudo("mkdir -p {prefix}".format(prefix=env.margin_engine_executor_prefix))
+ with cd("{prefix}".format(prefix=env.margin_engine_executor_prefix)):
+ sudo("wget https://s3.amazonaws.com/marvin-engine-executor/{jar}".format(jar=env.margin_engine_executor_jar))
+
+
+def create_marvin_engines_prefix():
+ sudo("mkdir -p /opt/marvin/engines")
+ sudo("chown {user}:{user} /opt/marvin/engines".format(user=env.user))
+ sudo("mkdir -p /var/log/marvin/engines")
+ sudo("chown {user}:{user} /var/log/marvin/engines".format(user=env.user))
+ sudo("mkdir -p /var/run/marvin/engines")
+ sudo("chown {user}:{user} /var/run/marvin/engines".format(user=env.user))
+
+
+def configure_marvin_environment():
+ run("echo 'export MARVIN_HOME=${HOME}/marvin' >> ${HOME}/.profile")
+ run("echo 'export MARVIN_DATA_PATH=${MARVIN_HOME}/data' >> ${HOME}/.profile")
+ run("mkdir -p ${MARVIN_HOME}")
+ run("mkdir -p ${MARVIN_DATA_PATH}")
+
+
+def provision():
+ execute(install_required_packages)
+ execute(install_virtualenvwrapper)
+ execute(install_oracle_jdk)
+ execute(install_apache_spark)
+ execute(install_marvin_engine_executor)
+ execute(create_marvin_engines_prefix)
+ execute(configure_marvin_environment)
+
+
+def package(version):
+ package = env.package
+ local("mkdir -p .packages")
+ local("tar czvf .packages/{package}-{version}.tar.gz --exclude='.packages' .".format(
+ package=package, version=version))
+
+
+def deploy(version, skip_clean=False):
+ execute(engine_stop)
+ package = env.package
+ put(local_path=".packages/{package}-{version}.tar.gz".format(
+ package=package, version=version), remote_path="/tmp/")
+ run("mkdir -p /opt/marvin/engines/{package}/{version}".format(
+ package=package, version=version))
+ with cd("/opt/marvin/engines/{package}/{version}".format(
+ package=package, version=version)):
+ run("tar xzvf /tmp/{package}-{version}.tar.gz".format(
+ package=package, version=version))
+ with cd("/opt/marvin/engines/{package}".format(package=package)):
+ symlink_exists = run("stat current", quiet=True).succeeded
+ if (symlink_exists):
+ run("rm current")
+ run("ln -s {version} current".format(version=version))
+ with cd("/opt/marvin/engines/{package}/current".format(package=package)):
+ run("mkvirtualenv {package}_env".format(package=package))
+ run("setvirtualenvproject")
+ if skip_clean:
+ run("workon {package}_env && make marvin".format(
+ package=package))
+ else:
+ run("workon {package}_env && make clean && make marvin".format(
+ package=package))
+ execute(engine_start)
+
+
+def engine_start(http_host, http_port):
+ package = env.package
+
+ command = (
+ "workon {package}_env &&"
+ " (marvin engine-httpserver"
+ " -h {http_host}"
+ " -p {http_port}"
+ " -e {executor}"
+ " 1> /var/log/marvin/engines/{package}.out"
+ " 2> /var/log/marvin/engines/{package}.err"
+ " & echo $! > /var/run/marvin/engines/{package}.pid)"
+ ).format(
+ package=package,
+ http_host=http_host,
+ http_port=http_port,
+ executor=env.marvin_engine_executor_path
+ )
+
+ with cd("/opt/marvin/engines/{package}/current".format(package=package)):
+ run(command, pty=False)
+
+
+def engine_stop():
+ package = env.package
+
+ pid_file_exists = run("cat /var/run/marvin/engines/{package}.pid".format(
+ package=package), quiet=True)
+ if pid_file_exists.succeeded:
+ with cd("/opt/marvin/engines/{package}/current".format(package=package)):
+ children_pids = run("ps --ppid $(cat /var/run/marvin/engines/{package}.pid) -o pid --no-headers |xargs echo".format(
+ package=package))
+ run("kill $(cat /var/run/marvin/engines/{package}.pid) {children_pids}".format(
+ package=package, children_pids=children_pids))
+ run("rm /var/run/marvin/engines/{package}.pid".format(package=package))
+
+
+def engine_status():
+ package = env.package
+ pid_file_exists = run("cat /var/run/marvin/engines/{package}.pid".format(
+ package=package), quiet=True)
+ if pid_file_exists.succeeded:
+ is_running = run("ps $(cat /var/run/marvin/engines/{package}.pid)".format(package=package), quiet=True)
+ if is_running.succeeded:
+ print "Your engine is running :)"
+ else:
+ print "Your engine is not running :("
+ else:
+ print "Your engine is not running :("
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/feedback.messages b/public-engines/iris-h2o-automl/feedback.messages
new file mode 100644
index 0000000..5ff7ec4
--- /dev/null
+++ b/public-engines/iris-h2o-automl/feedback.messages
@@ -0,0 +1,3 @@
+[{
+ "msg1": "Hello from marvin engine!"
+}]
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/install_automl.sh b/public-engines/iris-h2o-automl/install_automl.sh
new file mode 100644
index 0000000..4bde530
--- /dev/null
+++ b/public-engines/iris-h2o-automl/install_automl.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+###############################################
+# Marvin AutoML installation Script #
+###############################################
+cmd=(dialog --title "Marvin AutoML" --separate-output --checklist "Select tools:" 22 76 16)
+options=(1 "auto-sklearn" off # any option can be set to default to "on"
+ 2 "h2o AutoML" off
+ 3 "TPOT" off)
+choices=$("${cmd[@]}" "${options[@]}" 2>&1 >/dev/tty)
+clear
+for choice in $choices
+do
+ case $choice in
+ 1)
+ echo "Instaling auto-sklearn..."
+ wget https://raw.githubusercontent.com/automl/auto-sklearn/master/requirements.txt \
+ | xargs -n 1 -L 1 pip install
+ pip install auto-sklearn
+ ;;
+ 2)
+ echo "Installing h2o..."
+ pip install requests
+ pip install tabulate
+ pip install scikit-learn
+ pip install http://h2o-release.s3.amazonaws.com/h2o/rel-yau/3/Python/h2o-3.26.0.3-py2.py3-none-any.whl
+ wget http://h2o-release.s3.amazonaws.com/h2o/rel-yau/3/h2o-3.26.0.3.zip
+ unzip h2o-3.26.0.3.zip
+ rm h2o-3.26.0.3.zip
+ ;;
+ 3)
+ echo "Installing TPOT..."
+ pip install tpot
+ ;;
+ esac
+done
diff --git a/public-engines/iris-h2o-automl/marvin.ini b/public-engines/iris-h2o-automl/marvin.ini
new file mode 100644
index 0000000..f5b3b48
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin.ini
@@ -0,0 +1,11 @@
+[marvin]
+package = marvin_iris_h2o_automl
+type = automl-engine
+executor_url = https://s3.amazonaws.com/marvin-engine-executor/marvin-engine-executor-assembly-0.0.5.jar
+
+[ssh_deployment]
+# You can enable multi-host deployment like this
+# host = host1.com,host2.com,hostN.com
+host = host1.com
+port = 22
+user = marvin
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/VERSION b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/VERSION
new file mode 100644
index 0000000..8a9ecc2
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/VERSION
@@ -0,0 +1 @@
+0.0.1
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/__init__.py b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/__init__.py
new file mode 100644
index 0000000..e715c57
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/__init__.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os.path
+
+from .data_handler import *
+from .prediction import *
+from .training import *
+
+
+# Get package version number from "VERSION" file
+with open(os.path.join(os.path.dirname(__file__), 'VERSION'), 'rb') as f:
+ __version__ = f.read().decode('ascii').strip()
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/_compatibility.py b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/_compatibility.py
new file mode 100644
index 0000000..b2e635d
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/_compatibility.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Compatibility module.
+
+Import this module to help to write code compatible with Python 2 and 3.
+"""
+
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import six
+
+__all__ = ['six']
+
+# Add here any code that have to differentiate between python 2 and 3.
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/_logging.py b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/_logging.py
new file mode 100644
index 0000000..5f54346
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/_logging.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Custom logging module.
+
+This module is responsible to manage log messages and log file.
+"""
+
+import sys
+import os
+import os.path
+import logging
+
+DEFAULT_LOG_LEVEL = logging.WARNING
+DEFAULT_LOG_DIR = '/tmp'
+
+
+class Logger(logging.getLoggerClass()):
+ """Custom logger class.
+
+ Use this class to customize the logger behavior or to intercept the
+ messages.
+ """
+ def error(self, msg, *args, **kwargs):
+ # Add here code to intercept the project error messages
+ super(Logger, self).error(msg, *args, **kwargs)
+
+ def critical(self, msg, *args, **kwargs):
+ # Add here code to intercept the project critical messages
+ super(Logger, self).critical(msg, *args, **kwargs)
+
+
+logging.setLoggerClass(Logger)
+
+
+def get_logger(name, namespace='marvin_iris_h2o_automl',
+ log_level=DEFAULT_LOG_LEVEL, log_dir=DEFAULT_LOG_DIR):
+ """Build a logger that outputs to a file and to the console,"""
+
+ log_level = (os.getenv('{}_LOG_LEVEL'.format(namespace.upper())) or
+ os.getenv('LOG_LEVEL', log_level))
+ log_dir = (os.getenv('{}_LOG_DIR'.format(namespace.upper())) or
+ os.getenv('LOG_DIR', log_dir))
+
+ logger = logging.getLogger('{}.{}'.format(namespace, name))
+ logger.setLevel(log_level)
+
+ formatter = logging.Formatter(
+ '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+ # Create a console stream handler
+ console_handler = logging.StreamHandler()
+ console_handler.setLevel(log_level)
+ console_handler.setFormatter(formatter)
+ logger.addHandler(console_handler)
+
+ try:
+ if log_dir:
+ log_path = os.path.abspath(log_dir)
+ log_filename = '{name}.{pid}.log'.format(
+ name=namespace, pid=os.getpid())
+
+ file_path = str(os.path.join(log_path, log_filename))
+
+ if not os.path.exists(log_path):
+ os.makedirs(log_path, mode=774)
+
+ # Create a file handler
+ file_handler = logging.FileHandler(file_path)
+ file_handler.setLevel(log_level)
+ file_handler.setFormatter(formatter)
+ logger.addHandler(file_handler)
+ except OSError as e:
+ logger.error('Could not create log file {file}: {error}'.format(
+ file=file_path, error=e.strerror))
+
+ return logger
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/data_handler/__init__.py b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/data_handler/__init__.py
new file mode 100644
index 0000000..677dd17
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/data_handler/__init__.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+from .acquisitor_and_cleaner import AcquisitorAndCleaner
+from .training_preparator import TrainingPreparator
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/data_handler/acquisitor_and_cleaner.py b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/data_handler/acquisitor_and_cleaner.py
new file mode 100644
index 0000000..6cdecb0
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/data_handler/acquisitor_and_cleaner.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""AcquisitorAndCleaner engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBaseDataHandler
+
+__all__ = ['AcquisitorAndCleaner']
+
+
+logger = get_logger('acquisitor_and_cleaner')
+
+
+class AcquisitorAndCleaner(EngineBaseDataHandler):
+
+ def __init__(self, **kwargs):
+ super(AcquisitorAndCleaner, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ import pandas as pd
+ from marvin_python_toolbox.common.data import MarvinData
+
+ file_path = MarvinData.download_file(url="https://s3.amazonaws.com/marvin-engines-data/Iris.csv")
+
+ iris = pd.read_csv(file_path)
+
+ self.marvin_initial_dataset = iris
+
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/data_handler/training_preparator.py b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/data_handler/training_preparator.py
new file mode 100644
index 0000000..7d7bcb3
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/data_handler/training_preparator.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""TrainingPreparator engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBaseDataHandler
+
+__all__ = ['TrainingPreparator']
+
+
+logger = get_logger('training_preparator')
+
+
+class TrainingPreparator(EngineBaseDataHandler):
+
+ def __init__(self, **kwargs):
+ super(TrainingPreparator, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ from sklearn.model_selection import train_test_split
+ from sklearn import model_selection
+
+ X_train, X_test = train_test_split(self.marvin_initial_dataset, random_state=1, test_size=0.3)
+
+ self.marvin_dataset = {'train_X': X_train, 'test_X': X_test}
+
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/model_serializer.py b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/model_serializer.py
new file mode 100644
index 0000000..a7b4622
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/model_serializer.py
@@ -0,0 +1,19 @@
+import h2o
+import os
+
+class ModelSerializer(object):
+
+ def _serializer_load(self, object_file_path):
+ if object_file_path.split(os.sep)[-1] == 'model':
+ h2o.init()
+ return h2o.load_model(object_file_path)
+ else:
+ return super(ModelSerializer, self)._serializer_load(object_file_path)
+
+ def _serializer_dump(self, obj, object_file_path):
+ if object_file_path.split(os.sep)[-1] == 'model':
+ object_file_path = object_file_path[:-6]
+ h2o.save_model(model=obj.leader, path=object_file_path, force=True)
+ os.rename(object_file_path + '/' + obj.leader.model_id, object_file_path + '/model')
+ else:
+ super(ModelSerializer, self)._serializer_dump(obj, object_file_path)
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/prediction/__init__.py b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/prediction/__init__.py
new file mode 100644
index 0000000..e9b3c7c
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/prediction/__init__.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+from .prediction_preparator import PredictionPreparator
+from .predictor import Predictor
+from .feedback import Feedback
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/prediction/feedback.py b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/prediction/feedback.py
new file mode 100644
index 0000000..085808b
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/prediction/feedback.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2019] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Feedback engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBasePrediction
+
+__all__ = ['Feedback']
+
+
+logger = get_logger('feedback')
+
+
+class Feedback(EngineBasePrediction):
+
+ def __init__(self, **kwargs):
+ super(Feedback, self).__init__(**kwargs)
+
+ def execute(self, input_message, params, **kwargs):
+ """
+ Receive feedback message, user can manipulate this message for any use.
+ Return "Done" to signal that the message is received and processed.
+ """
+ return {"message": "Done"}
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/prediction/prediction_preparator.py b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/prediction/prediction_preparator.py
new file mode 100644
index 0000000..8394c07
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/prediction/prediction_preparator.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""PredictionPreparator engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBasePrediction
+
+__all__ = ['PredictionPreparator']
+
+
+logger = get_logger('prediction_preparator')
+
+
+class PredictionPreparator(EngineBasePrediction):
+
+ def __init__(self, **kwargs):
+ super(PredictionPreparator, self).__init__(**kwargs)
+
+ def execute(self, input_message, params, **kwargs):
+ import h2o
+ import pandas as pd
+
+ input_message = {'SepalLengthCm': [input_message[0]], 'SepalWidthCm': [input_message[1]],
+ 'PetalLengthCm': [input_message[2]], 'PetalWidthCm': [input_message[3]]}
+ input_message = pd.DataFrame(data=input_message)
+ input_message = h2o.H2OFrame.from_python(input_message)
+
+ return input_message
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/prediction/predictor.py b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/prediction/predictor.py
new file mode 100644
index 0000000..0a3c892
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/prediction/predictor.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Predictor engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBasePrediction
+from ..model_serializer import ModelSerializer
+
+__all__ = ['Predictor']
+
+
+logger = get_logger('predictor')
+
+
+class Predictor(ModelSerializer, EngineBasePrediction):
+
+ def __init__(self, **kwargs):
+ super(Predictor, self).__init__(**kwargs)
+
+ def execute(self, input_message, params, **kwargs):
+ final_prediction = self.marvin_model.predict(input_message).as_data_frame().values[0][0]
+
+ return final_prediction
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/training/__init__.py b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/training/__init__.py
new file mode 100644
index 0000000..e1723b7
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/training/__init__.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+from .metrics_evaluator import MetricsEvaluator
+from .trainer import Trainer
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/training/metrics_evaluator.py b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/training/metrics_evaluator.py
new file mode 100644
index 0000000..79c5709
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/training/metrics_evaluator.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""MetricsEvaluator engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBaseTraining
+from ..model_serializer import ModelSerializer
+
+__all__ = ['MetricsEvaluator']
+
+
+logger = get_logger('metrics_evaluator')
+
+
+class MetricsEvaluator(ModelSerializer, EngineBaseTraining):
+
+ def __init__(self, **kwargs):
+ super(MetricsEvaluator, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ import h2o
+ from sklearn import metrics
+
+ # h2o.init()
+
+ y_test = self.marvin_dataset['test_X']['Species']
+ self.marvin_dataset['test_X'].drop(columns='Species', inplace=True)
+
+ teste = h2o.H2OFrame.from_python(self.marvin_dataset['test_X'])
+ preds = self.marvin_model.predict(teste).as_data_frame()['predict'].values
+ self.marvin_metrics = metrics.accuracy_score(y_test, preds)
+
diff --git a/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/training/trainer.py b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/training/trainer.py
new file mode 100644
index 0000000..5443e68
--- /dev/null
+++ b/public-engines/iris-h2o-automl/marvin_iris_h2o_automl/training/trainer.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Trainer engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBaseTraining
+from ..model_serializer import ModelSerializer
+
+__all__ = ['Trainer']
+
+
+logger = get_logger('trainer')
+
+
+class Trainer(ModelSerializer, EngineBaseTraining):
+
+ def __init__(self, **kwargs):
+ super(Trainer, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ import h2o
+ from h2o.automl import H2OAutoML
+
+ h2o.init()
+
+ train_X_frame = h2o.H2OFrame.from_python(self.marvin_dataset['train_X'])
+ test_X_frame = h2o.H2OFrame.from_python(self.marvin_dataset['test_X'])
+
+ x = train_X_frame.columns
+ y = 'Species'
+ x.remove(y)
+
+ automl = H2OAutoML(max_models=20, seed=1)
+ automl.train(x=x,
+ y=y,
+ training_frame=train_X_frame)
+
+ self.marvin_model = automl
+
diff --git a/public-engines/iris-h2o-automl/notebooks/.ipynb_checkpoints/sample-checkpoint.ipynb b/public-engines/iris-h2o-automl/notebooks/.ipynb_checkpoints/sample-checkpoint.ipynb
new file mode 100644
index 0000000..6f8a801
--- /dev/null
+++ b/public-engines/iris-h2o-automl/notebooks/.ipynb_checkpoints/sample-checkpoint.ipynb
@@ -0,0 +1,318 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Iris H2O Example"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Acquisitor and Cleaner"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "marvin_cell": "acquisitor"
+ },
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "from marvin_python_toolbox.common.data import MarvinData\n",
+ "\n",
+ "file_path = MarvinData.download_file(url=\"https://s3.amazonaws.com/marvin-engines-data/Iris.csv\")\n",
+ "\n",
+ "iris = pd.read_csv(file_path)\n",
+ "\n",
+ "marvin_initial_dataset = iris"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Training Preparator"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "marvin_cell": "tpreparator"
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn.model_selection import train_test_split\n",
+ "from sklearn import model_selection\n",
+ "\n",
+ "X_train, X_test= train_test_split(marvin_initial_dataset, random_state=1,test_size=0.3)\n",
+ "\n",
+ "marvin_dataset = {'train_X': X_train, 'test_X': X_test}\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Trainer"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "marvin_cell": "trainer",
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Checking whether there is an H2O instance running at http://localhost:54321 . connected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
+ "<td>1 day 0 hours 48 mins</td></tr>\n",
+ "<tr><td>H2O cluster timezone:</td>\n",
+ "<td>America/Sao_Paulo</td></tr>\n",
+ "<tr><td>H2O data parsing timezone:</td>\n",
+ "<td>UTC</td></tr>\n",
+ "<tr><td>H2O cluster version:</td>\n",
+ "<td>3.26.0.3</td></tr>\n",
+ "<tr><td>H2O cluster version age:</td>\n",
+ "<td>20 days </td></tr>\n",
+ "<tr><td>H2O cluster name:</td>\n",
+ "<td>H2O_from_python_fernandozagatti_7kn410</td></tr>\n",
+ "<tr><td>H2O cluster total nodes:</td>\n",
+ "<td>1</td></tr>\n",
+ "<tr><td>H2O cluster free memory:</td>\n",
+ "<td>3.791 Gb</td></tr>\n",
+ "<tr><td>H2O cluster total cores:</td>\n",
+ "<td>4</td></tr>\n",
+ "<tr><td>H2O cluster allowed cores:</td>\n",
+ "<td>4</td></tr>\n",
+ "<tr><td>H2O cluster status:</td>\n",
+ "<td>locked, healthy</td></tr>\n",
+ "<tr><td>H2O connection url:</td>\n",
+ "<td>http://localhost:54321</td></tr>\n",
+ "<tr><td>H2O connection proxy:</td>\n",
+ "<td>None</td></tr>\n",
+ "<tr><td>H2O internal security:</td>\n",
+ "<td>False</td></tr>\n",
+ "<tr><td>H2O API Extensions:</td>\n",
+ "<td>Amazon S3, XGBoost, Algos, AutoML, Core V3, Core V4</td></tr>\n",
+ "<tr><td>Python version:</td>\n",
+ "<td>3.7.3 final</td></tr></table></div>"
+ ],
+ "text/plain": [
+ "-------------------------- ---------------------------------------------------\n",
+ "H2O cluster uptime: 1 day 0 hours 48 mins\n",
+ "H2O cluster timezone: America/Sao_Paulo\n",
+ "H2O data parsing timezone: UTC\n",
+ "H2O cluster version: 3.26.0.3\n",
+ "H2O cluster version age: 20 days\n",
+ "H2O cluster name: H2O_from_python_fernandozagatti_7kn410\n",
+ "H2O cluster total nodes: 1\n",
+ "H2O cluster free memory: 3.791 Gb\n",
+ "H2O cluster total cores: 4\n",
+ "H2O cluster allowed cores: 4\n",
+ "H2O cluster status: locked, healthy\n",
+ "H2O connection url: http://localhost:54321\n",
+ "H2O connection proxy:\n",
+ "H2O internal security: False\n",
+ "H2O API Extensions: Amazon S3, XGBoost, Algos, AutoML, Core V3, Core V4\n",
+ "Python version: 3.7.3 final\n",
+ "-------------------------- ---------------------------------------------------"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
+ "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
+ "AutoML progress: |████████████████████████████████████████████████████████| 100%\n"
+ ]
+ }
+ ],
+ "source": [
+ "import h2o\n",
+ "from h2o.automl import H2OAutoML\n",
+ "\n",
+ "h2o.init()\n",
+ "\n",
+ "train_X_frame = h2o.H2OFrame.from_python(marvin_dataset['train_X'])\n",
+ "test_X_frame = h2o.H2OFrame.from_python(marvin_dataset['test_X'])\n",
+ "\n",
+ "x = train_X_frame.columns\n",
+ "y = 'Species'\n",
+ "x.remove(y)\n",
+ "\n",
+ "automl = H2OAutoML(max_models=20, seed=1)\n",
+ "automl.train(x=x, \n",
+ " y=y, \n",
+ " training_frame=train_X_frame)\n",
+ "\n",
+ "marvin_model = automl"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Metrics Evaluator"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "marvin_cell": "evaluator",
+ "scrolled": false
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/fernandozagatti/.virtualenvs/iris-h2o-automl-env/lib/python3.7/site-packages/pandas/core/frame.py:4102: SettingWithCopyWarning: \n",
+ "A value is trying to be set on a copy of a slice from a DataFrame\n",
+ "\n",
+ "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+ " errors=errors,\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
+ "gbm prediction progress: |████████████████████████████████████████████████| 100%\n"
+ ]
+ }
+ ],
+ "source": [
+ "import h2o\n",
+ "from sklearn import metrics\n",
+ "\n",
+ "#h2o.init()\n",
+ "\n",
+ "y_test = marvin_dataset['test_X']['Species']\n",
+ "marvin_dataset['test_X'].drop(columns='Species',inplace= True)\n",
+ "\n",
+ "teste = h2o.H2OFrame.from_python(marvin_dataset['test_X'])\n",
+ "preds = marvin_model.predict(teste).as_data_frame()['predict'].values\n",
+ "marvin_metrics = metrics.accuracy_score(y_test, preds)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Prediction Preparator"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "marvin_cell": "ppreparator",
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
+ ]
+ }
+ ],
+ "source": [
+ "import h2o\n",
+ "import pandas as pd\n",
+ "\n",
+ "input_message = {'SepalLengthCm': [input_message[0]], 'SepalWidthCm': [input_message[1]],\n",
+ " 'PetalLengthCm': [input_message[2]], 'PetalWidthCm': [input_message[3]]}\n",
+ "input_message = pd.DataFrame(data=input_message)\n",
+ "input_message = h2o.H2OFrame.from_python(input_message)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Predictor"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "marvin_cell": "predictor"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "gbm prediction progress: |████████████████████████████████████████████████| 100%\n"
+ ]
+ }
+ ],
+ "source": [
+ "final_prediction = marvin_model.predict(input_message).as_data_frame().values[0][0]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Iris-versicolor\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(final_prediction)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/public-engines/iris-h2o-automl/notebooks/sample.ipynb b/public-engines/iris-h2o-automl/notebooks/sample.ipynb
new file mode 100644
index 0000000..6f8a801
--- /dev/null
+++ b/public-engines/iris-h2o-automl/notebooks/sample.ipynb
@@ -0,0 +1,318 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Iris H2O Example"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Acquisitor and Cleaner"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "marvin_cell": "acquisitor"
+ },
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "from marvin_python_toolbox.common.data import MarvinData\n",
+ "\n",
+ "file_path = MarvinData.download_file(url=\"https://s3.amazonaws.com/marvin-engines-data/Iris.csv\")\n",
+ "\n",
+ "iris = pd.read_csv(file_path)\n",
+ "\n",
+ "marvin_initial_dataset = iris"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Training Preparator"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "marvin_cell": "tpreparator"
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn.model_selection import train_test_split\n",
+ "from sklearn import model_selection\n",
+ "\n",
+ "X_train, X_test= train_test_split(marvin_initial_dataset, random_state=1,test_size=0.3)\n",
+ "\n",
+ "marvin_dataset = {'train_X': X_train, 'test_X': X_test}\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Trainer"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "marvin_cell": "trainer",
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Checking whether there is an H2O instance running at http://localhost:54321 . connected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
+ "<td>1 day 0 hours 48 mins</td></tr>\n",
+ "<tr><td>H2O cluster timezone:</td>\n",
+ "<td>America/Sao_Paulo</td></tr>\n",
+ "<tr><td>H2O data parsing timezone:</td>\n",
+ "<td>UTC</td></tr>\n",
+ "<tr><td>H2O cluster version:</td>\n",
+ "<td>3.26.0.3</td></tr>\n",
+ "<tr><td>H2O cluster version age:</td>\n",
+ "<td>20 days </td></tr>\n",
+ "<tr><td>H2O cluster name:</td>\n",
+ "<td>H2O_from_python_fernandozagatti_7kn410</td></tr>\n",
+ "<tr><td>H2O cluster total nodes:</td>\n",
+ "<td>1</td></tr>\n",
+ "<tr><td>H2O cluster free memory:</td>\n",
+ "<td>3.791 Gb</td></tr>\n",
+ "<tr><td>H2O cluster total cores:</td>\n",
+ "<td>4</td></tr>\n",
+ "<tr><td>H2O cluster allowed cores:</td>\n",
+ "<td>4</td></tr>\n",
+ "<tr><td>H2O cluster status:</td>\n",
+ "<td>locked, healthy</td></tr>\n",
+ "<tr><td>H2O connection url:</td>\n",
+ "<td>http://localhost:54321</td></tr>\n",
+ "<tr><td>H2O connection proxy:</td>\n",
+ "<td>None</td></tr>\n",
+ "<tr><td>H2O internal security:</td>\n",
+ "<td>False</td></tr>\n",
+ "<tr><td>H2O API Extensions:</td>\n",
+ "<td>Amazon S3, XGBoost, Algos, AutoML, Core V3, Core V4</td></tr>\n",
+ "<tr><td>Python version:</td>\n",
+ "<td>3.7.3 final</td></tr></table></div>"
+ ],
+ "text/plain": [
+ "-------------------------- ---------------------------------------------------\n",
+ "H2O cluster uptime: 1 day 0 hours 48 mins\n",
+ "H2O cluster timezone: America/Sao_Paulo\n",
+ "H2O data parsing timezone: UTC\n",
+ "H2O cluster version: 3.26.0.3\n",
+ "H2O cluster version age: 20 days\n",
+ "H2O cluster name: H2O_from_python_fernandozagatti_7kn410\n",
+ "H2O cluster total nodes: 1\n",
+ "H2O cluster free memory: 3.791 Gb\n",
+ "H2O cluster total cores: 4\n",
+ "H2O cluster allowed cores: 4\n",
+ "H2O cluster status: locked, healthy\n",
+ "H2O connection url: http://localhost:54321\n",
+ "H2O connection proxy:\n",
+ "H2O internal security: False\n",
+ "H2O API Extensions: Amazon S3, XGBoost, Algos, AutoML, Core V3, Core V4\n",
+ "Python version: 3.7.3 final\n",
+ "-------------------------- ---------------------------------------------------"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
+ "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
+ "AutoML progress: |████████████████████████████████████████████████████████| 100%\n"
+ ]
+ }
+ ],
+ "source": [
+ "import h2o\n",
+ "from h2o.automl import H2OAutoML\n",
+ "\n",
+ "h2o.init()\n",
+ "\n",
+ "train_X_frame = h2o.H2OFrame.from_python(marvin_dataset['train_X'])\n",
+ "test_X_frame = h2o.H2OFrame.from_python(marvin_dataset['test_X'])\n",
+ "\n",
+ "x = train_X_frame.columns\n",
+ "y = 'Species'\n",
+ "x.remove(y)\n",
+ "\n",
+ "automl = H2OAutoML(max_models=20, seed=1)\n",
+ "automl.train(x=x, \n",
+ " y=y, \n",
+ " training_frame=train_X_frame)\n",
+ "\n",
+ "marvin_model = automl"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Metrics Evaluator"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "marvin_cell": "evaluator",
+ "scrolled": false
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/fernandozagatti/.virtualenvs/iris-h2o-automl-env/lib/python3.7/site-packages/pandas/core/frame.py:4102: SettingWithCopyWarning: \n",
+ "A value is trying to be set on a copy of a slice from a DataFrame\n",
+ "\n",
+ "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+ " errors=errors,\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
+ "gbm prediction progress: |████████████████████████████████████████████████| 100%\n"
+ ]
+ }
+ ],
+ "source": [
+ "import h2o\n",
+ "from sklearn import metrics\n",
+ "\n",
+ "#h2o.init()\n",
+ "\n",
+ "y_test = marvin_dataset['test_X']['Species']\n",
+ "marvin_dataset['test_X'].drop(columns='Species',inplace= True)\n",
+ "\n",
+ "teste = h2o.H2OFrame.from_python(marvin_dataset['test_X'])\n",
+ "preds = marvin_model.predict(teste).as_data_frame()['predict'].values\n",
+ "marvin_metrics = metrics.accuracy_score(y_test, preds)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Prediction Preparator"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "marvin_cell": "ppreparator",
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
+ ]
+ }
+ ],
+ "source": [
+ "import h2o\n",
+ "import pandas as pd\n",
+ "\n",
+ "input_message = {'SepalLengthCm': [input_message[0]], 'SepalWidthCm': [input_message[1]],\n",
+ " 'PetalLengthCm': [input_message[2]], 'PetalWidthCm': [input_message[3]]}\n",
+ "input_message = pd.DataFrame(data=input_message)\n",
+ "input_message = h2o.H2OFrame.from_python(input_message)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Predictor"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "marvin_cell": "predictor"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "gbm prediction progress: |████████████████████████████████████████████████| 100%\n"
+ ]
+ }
+ ],
+ "source": [
+ "final_prediction = marvin_model.predict(input_message).as_data_frame().values[0][0]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Iris-versicolor\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(final_prediction)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/public-engines/iris-h2o-automl/pytest.ini b/public-engines/iris-h2o-automl/pytest.ini
new file mode 100644
index 0000000..cab8644
--- /dev/null
+++ b/public-engines/iris-h2o-automl/pytest.ini
@@ -0,0 +1,4 @@
+[pytest]
+minversion = 2.0
+norecursedirs = .git .tox .eggs .cache *.egg build dist tmp*
+python_files = test*.py
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/requirements.txt b/public-engines/iris-h2o-automl/requirements.txt
new file mode 100644
index 0000000..983d39a
--- /dev/null
+++ b/public-engines/iris-h2o-automl/requirements.txt
@@ -0,0 +1,20 @@
+setuptools
+nose
+Cython
+
+numpy>=1.9.0
+scipy>=0.14.1
+
+scikit-learn>=0.21.0,<0.22
+
+lockfile
+joblib
+psutil
+pyyaml
+liac-arff
+pandas
+
+ConfigSpace>=0.4.0,<0.5
+pynisher>=0.4.2
+pyrfr>=0.7,<0.9
+smac==0.8
diff --git a/public-engines/iris-h2o-automl/setup.cfg b/public-engines/iris-h2o-automl/setup.cfg
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/public-engines/iris-h2o-automl/setup.cfg
diff --git a/public-engines/iris-h2o-automl/setup.py b/public-engines/iris-h2o-automl/setup.py
new file mode 100644
index 0000000..647b541
--- /dev/null
+++ b/public-engines/iris-h2o-automl/setup.py
@@ -0,0 +1,173 @@
+from __future__ import print_function
+
+import os
+import shutil
+from os.path import dirname, join
+from setuptools import setup, find_packages
+from setuptools.command.test import test as TestCommand
+from setuptools.command.develop import develop as _develop
+from setuptools.command.install import install as _install
+
+
+REQUIREMENTS_TESTS = [
+ 'pytest>=2.6.4',
+ 'pytest-cov>=1.8.1',
+ 'mock>=2.0.0',
+ 'virtualenv>=15.0.1',
+ 'tox>=2.2.0',
+]
+
+def _get_version():
+ """Return the project version from VERSION file."""
+
+ with open(join(dirname(__file__), 'marvin_iris_h2o_automl/VERSION'), 'rb') as f:
+ version = f.read().decode('ascii').strip()
+ return version
+
+
+def _hooks(dir):
+ _set_autocomplete()
+ _install_notebook_extension()
+
+def _set_autocomplete():
+ import marvin_python_toolbox as toolbox
+ virtualenv = os.environ.get('VIRTUAL_ENV', None)
+
+ if virtualenv:
+ postactivate = os.path.join(virtualenv, 'bin', 'postactivate')
+
+ if os.path.exists(postactivate):
+ shutil.copy(
+ os.path.join(toolbox.__path__[0], 'extras', 'marvin_bash_completion'),
+ os.path.join(virtualenv, 'marvin_bash_completion')
+ )
+
+ command = 'source "{}"'.format(os.path.join(virtualenv, 'marvin_bash_completion'))
+
+ with open(postactivate, 'r+') as fp:
+ lines = fp.readlines()
+ fp.seek(0)
+ configured = False
+ for line in lines:
+ if 'marvin_bash_completion' in line:
+ # Replacing old autocomplete configuration
+ fp.write(command)
+ configured = True
+ else:
+ fp.write(line)
+
+ if not configured:
+ fp.write(command)
+ # 'Autocomplete was successfully configured'
+ fp.write('\n')
+ fp.truncate()
+
+def _install_notebook_extension():
+ import marvin_python_toolbox as toolbox
+
+ install_command = [
+ "jupyter",
+ "nbextension",
+ "install",
+ os.path.join(toolbox.__path__[0], 'extras', 'notebook_extensions', 'main.js'),
+ "--destination",
+ "marvin.js",
+ "--sys-prefix",
+ "--overwrite"
+ ]
+
+ os.system(' '.join(install_command))
+
+ enable_command = [
+ "jupyter",
+ "nbextension",
+ "enable",
+ "marvin",
+ "--sys-prefix"
+ ]
+
+ os.system(' '.join(enable_command))
+
+
+class develop(_develop):
+ def run(self):
+ _develop.run(self)
+ self.execute(_hooks, (self.install_lib,), msg="Running develop preparation task")
+
+
+class install(_install):
+ def run(self):
+ _install.run(self)
+ self.execute(_hooks, (self.install_lib,), msg="Running install preparation task")
+
+
+class Tox(TestCommand):
+ """Run the test cases using TOX command."""
+
+ user_options = [('tox-args=', 'a', "Arguments to pass to tox")]
+
+ def initialize_options(self):
+ TestCommand.initialize_options(self)
+ self.tox_args = None
+
+ def finalize_options(self):
+ TestCommand.finalize_options(self)
+ self.test_args = []
+ self.test_suite = True
+
+ def run_tests(self):
+ # Import here, cause outside the eggs aren't loaded
+ import tox
+ import shlex
+ import sys
+ args = self.tox_args
+ if args:
+ args = shlex.split(self.tox_args)
+ else:
+ # Run all tests by default
+ args = ['-c', join(dirname(__file__), 'tox.ini'), 'tests']
+ errno = tox.cmdline(args=args)
+ sys.exit(errno)
+
+
+setup(
+ name='marvin_iris_h2o_automl',
+ version=_get_version(),
+ url='',
+ description='Marvin engine',
+ long_description=open(join(dirname(__file__), 'README.md')).read(),
+ author='Marvin AI Community',
+ maintainer='Marvin AI Community',
+ maintainer_email='dev@marvin.apache.org',
+ packages=find_packages(exclude=('tests', 'tests.*')),
+ include_package_data=True,
+ zip_safe=False,
+ classifiers=[
+ 'Development Status :: 3 - Alpha',
+ 'Intended Audience :: Developers',
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.3',
+ 'Programming Language :: Python :: 3.4',
+ 'Programming Language :: Python :: 3.5',
+ 'Topic :: Software Development :: Libraries :: Python Modules',
+ ],
+ install_requires=[
+ 'scikit-learn>=0.18.2',
+ 'scipy>=0.19.1',
+ 'numpy>=1.13.1',
+ 'pandas>=0.20.3',
+ 'matplotlib>=2.0.2',
+ 'marvin-python-toolbox>=0.0.5',
+ 'Fabric>=1.14.0',
+ ],
+ tests_require=REQUIREMENTS_TESTS,
+ extras_require={
+ 'testing': REQUIREMENTS_TESTS,
+ },
+ cmdclass={
+ 'test': Tox, 'develop': develop, 'install': install
+ },
+)
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/tests/conftest.py b/public-engines/iris-h2o-automl/tests/conftest.py
new file mode 100644
index 0000000..efd37f3
--- /dev/null
+++ b/public-engines/iris-h2o-automl/tests/conftest.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os
+import pytest
+
+os.environ['TESTING'] = 'True'
+
+
+@pytest.fixture
+def mocked_params():
+ return {'params': 1}
diff --git a/public-engines/iris-h2o-automl/tests/data_handler/test_acquisitor_and_cleaner.py b/public-engines/iris-h2o-automl/tests/data_handler/test_acquisitor_and_cleaner.py
new file mode 100644
index 0000000..ee6c0d5
--- /dev/null
+++ b/public-engines/iris-h2o-automl/tests/data_handler/test_acquisitor_and_cleaner.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_h2o_automl.data_handler import AcquisitorAndCleaner
+
+
+class TestAcquisitorAndCleaner:
+ def test_execute(self, mocked_params):
+ ac = AcquisitorAndCleaner()
+ ac.execute(params=mocked_params)
+ assert not ac._params
diff --git a/public-engines/iris-h2o-automl/tests/data_handler/test_training_preparator.py b/public-engines/iris-h2o-automl/tests/data_handler/test_training_preparator.py
new file mode 100644
index 0000000..3102767
--- /dev/null
+++ b/public-engines/iris-h2o-automl/tests/data_handler/test_training_preparator.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_h2o_automl.data_handler import TrainingPreparator
+
+
+class TestTrainingPreparator:
+ def test_execute(self, mocked_params):
+ ac = TrainingPreparator()
+ ac.execute(params=mocked_params)
+ assert not ac._params
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/tests/prediction/test_feedback.py b/public-engines/iris-h2o-automl/tests/prediction/test_feedback.py
new file mode 100644
index 0000000..07bb935
--- /dev/null
+++ b/public-engines/iris-h2o-automl/tests/prediction/test_feedback.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_h2o_automl.prediction import Feedback
+
+
+class TestFeedback:
+ def test_execute(self, mocked_params):
+ fb = Feedback()
+ fb.execute(input_message="fake message", params=mocked_params)
+ assert not fb._params
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/tests/prediction/test_prediction_preparator.py b/public-engines/iris-h2o-automl/tests/prediction/test_prediction_preparator.py
new file mode 100644
index 0000000..e0c501a
--- /dev/null
+++ b/public-engines/iris-h2o-automl/tests/prediction/test_prediction_preparator.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_h2o_automl.prediction import PredictionPreparator
+
+
+class TestPredictionPreparator:
+ def test_execute(self, mocked_params):
+ ac = PredictionPreparator()
+ ac.execute(input_message="fake message", params=mocked_params)
+ assert not ac._params
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/tests/prediction/test_predictor.py b/public-engines/iris-h2o-automl/tests/prediction/test_predictor.py
new file mode 100644
index 0000000..04c6e6a
--- /dev/null
+++ b/public-engines/iris-h2o-automl/tests/prediction/test_predictor.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_h2o_automl.prediction import Predictor
+
+
+class TestPredictor:
+ def test_execute(self, mocked_params):
+ ac = Predictor()
+ ac.execute(input_message="fake message", params=mocked_params)
+ assert not ac._params
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/tests/training/test_metrics_evaluator.py b/public-engines/iris-h2o-automl/tests/training/test_metrics_evaluator.py
new file mode 100644
index 0000000..018506a
--- /dev/null
+++ b/public-engines/iris-h2o-automl/tests/training/test_metrics_evaluator.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_h2o_automl.training import MetricsEvaluator
+
+
+class TestMetricsEvaluator:
+ def test_execute(self, mocked_params):
+ ac = MetricsEvaluator()
+ ac.execute(params=mocked_params)
+ assert not ac._params
\ No newline at end of file
diff --git a/public-engines/iris-h2o-automl/tests/training/test_trainer.py b/public-engines/iris-h2o-automl/tests/training/test_trainer.py
new file mode 100644
index 0000000..2047332
--- /dev/null
+++ b/public-engines/iris-h2o-automl/tests/training/test_trainer.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_h2o_automl.training import Trainer
+
+
+class TestTrainer:
+ def test_execute(self, mocked_params):
+ ac = Trainer()
+ ac.execute(params=mocked_params)
+ assert not ac._params
diff --git a/public-engines/iris-h2o-automl/tox.ini b/public-engines/iris-h2o-automl/tox.ini
new file mode 100644
index 0000000..46514bf
--- /dev/null
+++ b/public-engines/iris-h2o-automl/tox.ini
@@ -0,0 +1,8 @@
+[tox]
+envlist = py27
+
+[testenv]
+deps=pytest
+ pytest-cov
+ mock
+commands=py.test --cov={envsitepackagesdir}/marvin_iris_h2o_automl --cov-report html --cov-report xml {posargs}
\ No newline at end of file
diff --git a/public-engines/iris-species-engine/Makefile b/public-engines/iris-species-engine/Makefile
index 984ac42..0869661 100644
--- a/public-engines/iris-species-engine/Makefile
+++ b/public-engines/iris-species-engine/Makefile
@@ -1,4 +1,4 @@
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/iris-species-engine/docs.yaml b/public-engines/iris-species-engine/docs.yaml
index 0d1116f..546d3ec 100644
--- a/public-engines/iris-species-engine/docs.yaml
+++ b/public-engines/iris-species-engine/docs.yaml
@@ -4,8 +4,8 @@
title: marvin_iris_species_engine API Doc
contact:
name: mantainer Daniel Takabayashi
- email: daniel.takabayashi@b2wdigital.com
- url: https://github.com/marvin-ai
+ email: dev@marvin.apache.org
+ url: https://marvin.apache.org
license:
name: Apache License 2.0
servers:
diff --git a/public-engines/iris-species-engine/marvin_iris_species_engine/prediction/feedback.py b/public-engines/iris-species-engine/marvin_iris_species_engine/prediction/feedback.py
index bed51c1..4f926b1 100644
--- a/public-engines/iris-species-engine/marvin_iris_species_engine/prediction/feedback.py
+++ b/public-engines/iris-species-engine/marvin_iris_species_engine/prediction/feedback.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/iris-species-engine/setup.py b/public-engines/iris-species-engine/setup.py
index 2d86e4c..bd2da8a 100644
--- a/public-engines/iris-species-engine/setup.py
+++ b/public-engines/iris-species-engine/setup.py
@@ -58,7 +58,7 @@
long_description=open(join(dirname(__file__), 'README.md')).read(),
author='Marvin AI Researcher',
maintainer='Daniel Takabayashi',
- maintainer_email='daniel.takabayashi@gmail.com',
+ maintainer_email='dev@marvin.apache.org',
packages=find_packages(exclude=('tests', 'tests.*')),
include_package_data=True,
zip_safe=False,
@@ -83,7 +83,7 @@
'marvin-python-toolbox==0',
'seaborn==0.8.1'
],
- dependency_links=['git+https://github.com/marvin-ai/marvin-python-toolbox.git/@master#egg=marvin_python_toolbox-0'],
+ dependency_links=['git+https://github.com/apache/incubator-marvin@master#subdirectory=python-toolbox#egg=marvin_python_toolbox-0'],
tests_require=REQUIREMENT_TESTS,
extras_require={
'testing': REQUIREMENT_TESTS,
diff --git a/public-engines/kaggle-titanic-engine/Dockerfile b/public-engines/kaggle-titanic-engine/Dockerfile
index 3f7107d..4f0a31e 100644
--- a/public-engines/kaggle-titanic-engine/Dockerfile
+++ b/public-engines/kaggle-titanic-engine/Dockerfile
@@ -1,6 +1,7 @@
FROM ubuntu:16.04
-MAINTAINER jeremy.elster@b2wdigital.com
+MAINTAINER dev@marvin.apache.org
+LABEL authors="jeremy.elster"
ENV SLEEP_MILLIS 0
@@ -74,12 +75,6 @@
# spark configuration files to the think works correctly.
##############################################################
-# RUN curl https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz -o /tmp/spark-2.1.1-bin-hadoop2.6.tgz && \
-# tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/ && \
-# ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark
-
-# Add the b2w datalake config for Spark
-# ADD spark-conf.tar $SPARK_CONF_DIR
RUN mkdir -p $SPARK_CONF_DIR
##############################################################
diff --git a/public-engines/kaggle-titanic-engine/docs.yaml b/public-engines/kaggle-titanic-engine/docs.yaml
index 993e8c8..2941f51 100644
--- a/public-engines/kaggle-titanic-engine/docs.yaml
+++ b/public-engines/kaggle-titanic-engine/docs.yaml
@@ -4,8 +4,8 @@
title: marvin_titanic_engine API Doc
contact:
name: mantainer jeremy.elster
- email: jeremy.elster@b2wdigital.com
- url: https://github.com/marvin-ai
+ email: dev@marvin.apache.org
+ url: https://marvin.apache.org
license:
name: Apache License 2.0
servers:
diff --git a/public-engines/kaggle-titanic-engine/marvin_titanic_engine/prediction/feedback.py b/public-engines/kaggle-titanic-engine/marvin_titanic_engine/prediction/feedback.py
index ca42364..085808b 100644
--- a/public-engines/kaggle-titanic-engine/marvin_titanic_engine/prediction/feedback.py
+++ b/public-engines/kaggle-titanic-engine/marvin_titanic_engine/prediction/feedback.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/kaggle-titanic-engine/setup.py b/public-engines/kaggle-titanic-engine/setup.py
index 9a74b3f..04a7da1 100644
--- a/public-engines/kaggle-titanic-engine/setup.py
+++ b/public-engines/kaggle-titanic-engine/setup.py
@@ -58,8 +58,8 @@
description='a look at the kaggle data fr the titanic',
long_description=open(join(dirname(__file__), 'README.md')).read(),
author='jeremy.elster',
- maintainer='jeremy.elster',
- maintainer_email='jeremy.elster@b2wdigital.com',
+ maintainer='Apache Software Foundation',
+ maintainer_email='dev@marvin.apache.org',
packages=find_packages(exclude=('tests', 'tests.*')),
include_package_data=True,
zip_safe=False,
@@ -84,7 +84,7 @@
'marvin-python-toolbox==0',
'Fabric==1.14.0',
],
- dependency_links=['git+https://github.com/marvin-ai/marvin-python-toolbox.git/@master#egg=marvin_python_toolbox-0'],
+ dependency_links=['git+https://github.com/apache/incubator-marvin@master#subdirectory=python-toolbox#egg=marvin_python_toolbox-0'],
tests_require=REQUIREMENT_TESTS,
extras_require={
'testing': REQUIREMENT_TESTS,
diff --git a/public-engines/mnist-keras-engine/Dockerfile b/public-engines/mnist-keras-engine/Dockerfile
index 9ba1140..ed27aaf 100644
--- a/public-engines/mnist-keras-engine/Dockerfile
+++ b/public-engines/mnist-keras-engine/Dockerfile
@@ -1,6 +1,6 @@
FROM ubuntu:16.04
-MAINTAINER marvin-ai@googlegroups.com
+MAINTAINER dev@marvin.apache.org
ENV SLEEP_MILLIS 0
@@ -74,12 +74,6 @@
# spark configuration files to the think works correctly.
##############################################################
-# RUN curl https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz -o /tmp/spark-2.1.1-bin-hadoop2.6.tgz && \
-# tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/ && \
-# ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark
-
-# Add the b2w datalake config for Spark
-# ADD spark-conf.tar $SPARK_CONF_DIR
RUN mkdir -p $SPARK_CONF_DIR
##############################################################
diff --git a/public-engines/mnist-keras-engine/docs.yaml b/public-engines/mnist-keras-engine/docs.yaml
index 6e5b065..7f42a05 100644
--- a/public-engines/mnist-keras-engine/docs.yaml
+++ b/public-engines/mnist-keras-engine/docs.yaml
@@ -4,8 +4,8 @@
title: marvin_mnist_keras_engine API Doc
contact:
name: mantainer Marvin AI Community
- email: marvin-ai@googlegroups.com
- url: https://github.com/marvin-ai
+ email: dev@marvin.apache.org
+ url: https://marvin.apache.org
license:
name: Apache License 2.0
servers:
diff --git a/public-engines/mnist-keras-engine/marvin_mnist_keras_engine/prediction/feedback.py b/public-engines/mnist-keras-engine/marvin_mnist_keras_engine/prediction/feedback.py
index ca42364..085808b 100644
--- a/public-engines/mnist-keras-engine/marvin_mnist_keras_engine/prediction/feedback.py
+++ b/public-engines/mnist-keras-engine/marvin_mnist_keras_engine/prediction/feedback.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/mnist-keras-engine/setup.py b/public-engines/mnist-keras-engine/setup.py
index 5dab442..d846a35 100644
--- a/public-engines/mnist-keras-engine/setup.py
+++ b/public-engines/mnist-keras-engine/setup.py
@@ -59,7 +59,7 @@
long_description=open(join(dirname(__file__), 'README.md')).read(),
author='Marvin AI Community',
maintainer='Marvin AI Community',
- maintainer_email='marvin-ai@googlegroups.com',
+ maintainer_email='dev@marvin.apache.org',
packages=find_packages(exclude=('tests', 'tests.*')),
include_package_data=True,
zip_safe=False,
@@ -90,7 +90,7 @@
'pluggy==0.3.1',
'opencv-python==3.4.0.12'
],
- dependency_links=['git+https://github.com/marvin-ai/marvin-python-toolbox.git/@master#egg=marvin_python_toolbox-0'],
+ dependency_links=['git+https://github.com/apache/incubator-marvin@master#subdirectory=python-toolbox#egg=marvin_python_toolbox-0'],
tests_require=REQUIREMENT_TESTS,
extras_require={
'testing': REQUIREMENT_TESTS,
diff --git a/public-engines/nlp-ner-engine/Dockerfile b/public-engines/nlp-ner-engine/Dockerfile
index 8229e40..edcaba6 100644
--- a/public-engines/nlp-ner-engine/Dockerfile
+++ b/public-engines/nlp-ner-engine/Dockerfile
@@ -1,6 +1,6 @@
FROM ubuntu:16.04
-MAINTAINER marvin-ai@googlegroups.com
+MAINTAINER dev@marvin.apache.org
ENV SLEEP_MILLIS 0
@@ -74,12 +74,6 @@
# spark configuration files to the think works correctly.
##############################################################
-# RUN curl https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz -o /tmp/spark-2.1.1-bin-hadoop2.6.tgz && \
-# tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/ && \
-# ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark
-
-# Add the b2w datalake config for Spark
-# ADD spark-conf.tar $SPARK_CONF_DIR
RUN mkdir -p $SPARK_CONF_DIR
##############################################################
diff --git a/public-engines/nlp-ner-engine/docs.yaml b/public-engines/nlp-ner-engine/docs.yaml
index c08d924..1b5062f 100644
--- a/public-engines/nlp-ner-engine/docs.yaml
+++ b/public-engines/nlp-ner-engine/docs.yaml
@@ -4,8 +4,8 @@
title: marvin_nlp_ner_engine API Doc
contact:
name: mantainer Marvin AI Community
- email: marvin-ai@googlegroups.com
- url: https://github.com/marvin-ai
+ email: dev@marvin.apache.org
+ url: https://marvin.apache.org
license:
name: Apache License 2.0
servers:
diff --git a/public-engines/nlp-ner-engine/marvin_nlp_ner_engine/prediction/feedback.py b/public-engines/nlp-ner-engine/marvin_nlp_ner_engine/prediction/feedback.py
index ca42364..085808b 100644
--- a/public-engines/nlp-ner-engine/marvin_nlp_ner_engine/prediction/feedback.py
+++ b/public-engines/nlp-ner-engine/marvin_nlp_ner_engine/prediction/feedback.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/nlp-ner-engine/setup.py b/public-engines/nlp-ner-engine/setup.py
index 845e26d..8223d7b 100644
--- a/public-engines/nlp-ner-engine/setup.py
+++ b/public-engines/nlp-ner-engine/setup.py
@@ -75,7 +75,7 @@
'Programming Language :: Python :: 3.5',
'Topic :: Software Development :: Libraries :: Python Modules',
],
- dependency_links=['git+https://github.com/marvin-ai/marvin-python-toolbox.git/@master#egg=marvin_python_toolbox-0'],
+ dependency_links=['git+https://github.com/apache/incubator-marvin@master#subdirectory=python-toolbox#egg=marvin_python_toolbox-0'],
install_requires=[
'scikit-learn==0.18.2',
'scipy==0.19.1',
diff --git a/public-engines/product-classifier-engine/Dockerfile b/public-engines/product-classifier-engine/Dockerfile
index 9ba1140..ed27aaf 100644
--- a/public-engines/product-classifier-engine/Dockerfile
+++ b/public-engines/product-classifier-engine/Dockerfile
@@ -1,6 +1,6 @@
FROM ubuntu:16.04
-MAINTAINER marvin-ai@googlegroups.com
+MAINTAINER dev@marvin.apache.org
ENV SLEEP_MILLIS 0
@@ -74,12 +74,6 @@
# spark configuration files to the think works correctly.
##############################################################
-# RUN curl https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz -o /tmp/spark-2.1.1-bin-hadoop2.6.tgz && \
-# tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/ && \
-# ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark
-
-# Add the b2w datalake config for Spark
-# ADD spark-conf.tar $SPARK_CONF_DIR
RUN mkdir -p $SPARK_CONF_DIR
##############################################################
diff --git a/public-engines/product-classifier-engine/docs.yaml b/public-engines/product-classifier-engine/docs.yaml
index 9e7e4ac..46ae8d6 100644
--- a/public-engines/product-classifier-engine/docs.yaml
+++ b/public-engines/product-classifier-engine/docs.yaml
@@ -4,8 +4,8 @@
title: marvin_product_classifier_engine API Doc
contact:
name: mantainer Marvin AI Community
- email: marvin-ai@googlegroups.com
- url: https://github.com/marvin-ai
+ email: dev@marvin.apache.org
+ url: https://marvin.apache.org
license:
name: Apache License 2.0
servers:
diff --git a/public-engines/product-classifier-engine/marvin_product_classifier_engine/prediction/feedback.py b/public-engines/product-classifier-engine/marvin_product_classifier_engine/prediction/feedback.py
index ca42364..085808b 100644
--- a/public-engines/product-classifier-engine/marvin_product_classifier_engine/prediction/feedback.py
+++ b/public-engines/product-classifier-engine/marvin_product_classifier_engine/prediction/feedback.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/product-classifier-engine/setup.py b/public-engines/product-classifier-engine/setup.py
index 544dfac..1ad8daa 100644
--- a/public-engines/product-classifier-engine/setup.py
+++ b/public-engines/product-classifier-engine/setup.py
@@ -59,7 +59,7 @@
long_description=open(join(dirname(__file__), 'README.md')).read(),
author='Marvin AI Community',
maintainer='Marvin AI Community',
- maintainer_email='marvin-ai@googlegroups.com',
+ maintainer_email='dev@marvin.apache.org',
packages=find_packages(exclude=('tests', 'tests.*')),
include_package_data=True,
zip_safe=False,
@@ -84,7 +84,7 @@
'marvin-python-toolbox==0',
'Fabric==1.14.0',
],
- dependency_links=['git+https://github.com/marvin-ai/marvin-python-toolbox.git/@master#egg=marvin_python_toolbox-0'],
+ dependency_links=['git+https://github.com/apache/incubator-marvin@master#subdirectory=python-toolbox#egg=marvin_python_toolbox-0'],
tests_require=REQUIREMENT_TESTS,
extras_require={
'testing': REQUIREMENT_TESTS,
diff --git a/public-engines/sample-engine/Makefile b/public-engines/sample-engine/Makefile
index e18ffb8..396df8d 100644
--- a/public-engines/sample-engine/Makefile
+++ b/public-engines/sample-engine/Makefile
@@ -1,4 +1,4 @@
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/marvin_sample_engine/__init__.py b/public-engines/sample-engine/marvin_sample_engine/__init__.py
index e073afc..b209757 100644
--- a/public-engines/sample-engine/marvin_sample_engine/__init__.py
+++ b/public-engines/sample-engine/marvin_sample_engine/__init__.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/marvin_sample_engine/_compatibility.py b/public-engines/sample-engine/marvin_sample_engine/_compatibility.py
index 4414bd1..caa3fab 100644
--- a/public-engines/sample-engine/marvin_sample_engine/_compatibility.py
+++ b/public-engines/sample-engine/marvin_sample_engine/_compatibility.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/marvin_sample_engine/_logging.py b/public-engines/sample-engine/marvin_sample_engine/_logging.py
index 2a63460..2944beb 100644
--- a/public-engines/sample-engine/marvin_sample_engine/_logging.py
+++ b/public-engines/sample-engine/marvin_sample_engine/_logging.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/marvin_sample_engine/data_handler/__init__.py b/public-engines/sample-engine/marvin_sample_engine/data_handler/__init__.py
index 79b921b..084bf14 100644
--- a/public-engines/sample-engine/marvin_sample_engine/data_handler/__init__.py
+++ b/public-engines/sample-engine/marvin_sample_engine/data_handler/__init__.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/marvin_sample_engine/data_handler/acquisitor_and_cleaner.py b/public-engines/sample-engine/marvin_sample_engine/data_handler/acquisitor_and_cleaner.py
index 8ac2381..50a7555 100644
--- a/public-engines/sample-engine/marvin_sample_engine/data_handler/acquisitor_and_cleaner.py
+++ b/public-engines/sample-engine/marvin_sample_engine/data_handler/acquisitor_and_cleaner.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/marvin_sample_engine/data_handler/training_preparator.py b/public-engines/sample-engine/marvin_sample_engine/data_handler/training_preparator.py
index 8d2e878..f0ebb2b 100644
--- a/public-engines/sample-engine/marvin_sample_engine/data_handler/training_preparator.py
+++ b/public-engines/sample-engine/marvin_sample_engine/data_handler/training_preparator.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/marvin_sample_engine/prediction/__init__.py b/public-engines/sample-engine/marvin_sample_engine/prediction/__init__.py
index fddd35e..89f2380 100644
--- a/public-engines/sample-engine/marvin_sample_engine/prediction/__init__.py
+++ b/public-engines/sample-engine/marvin_sample_engine/prediction/__init__.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/marvin_sample_engine/prediction/prediction_preparator.py b/public-engines/sample-engine/marvin_sample_engine/prediction/prediction_preparator.py
index 2076ef1..3c1e602 100644
--- a/public-engines/sample-engine/marvin_sample_engine/prediction/prediction_preparator.py
+++ b/public-engines/sample-engine/marvin_sample_engine/prediction/prediction_preparator.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/marvin_sample_engine/prediction/predictor.py b/public-engines/sample-engine/marvin_sample_engine/prediction/predictor.py
index 17c23a3..00f2991 100644
--- a/public-engines/sample-engine/marvin_sample_engine/prediction/predictor.py
+++ b/public-engines/sample-engine/marvin_sample_engine/prediction/predictor.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/marvin_sample_engine/training/__init__.py b/public-engines/sample-engine/marvin_sample_engine/training/__init__.py
index 804382b..8bda353 100644
--- a/public-engines/sample-engine/marvin_sample_engine/training/__init__.py
+++ b/public-engines/sample-engine/marvin_sample_engine/training/__init__.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/marvin_sample_engine/training/metrics_evaluator.py b/public-engines/sample-engine/marvin_sample_engine/training/metrics_evaluator.py
index bb19159..5abca15 100644
--- a/public-engines/sample-engine/marvin_sample_engine/training/metrics_evaluator.py
+++ b/public-engines/sample-engine/marvin_sample_engine/training/metrics_evaluator.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/marvin_sample_engine/training/trainer.py b/public-engines/sample-engine/marvin_sample_engine/training/trainer.py
index 4abb1f9..e8e80a4 100644
--- a/public-engines/sample-engine/marvin_sample_engine/training/trainer.py
+++ b/public-engines/sample-engine/marvin_sample_engine/training/trainer.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/setup.py b/public-engines/sample-engine/setup.py
index 91f84c1..dd5742c 100644
--- a/public-engines/sample-engine/setup.py
+++ b/public-engines/sample-engine/setup.py
@@ -1,5 +1,5 @@
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -64,8 +64,8 @@
description='Marvin engine',
long_description=open(join(dirname(__file__), 'README.md')).read(),
author='Marvin AI Researcher',
- maintainer='B2W Labs Team',
- maintainer_email='@b2wdigital.com',
+ maintainer='Apache Software Foundation',
+ maintainer_email='dev@marvin.apache.org',
packages=find_packages(exclude=('tests', 'tests.*')),
include_package_data=True,
zip_safe=False,
@@ -90,7 +90,7 @@
'matplotlib==2.0.2',
'marvin-python-toolbox==0',
],
- dependency_links=['git+https://github.com/marvin-ai/marvin-python-toolbox.git/@master#egg=marvin_python_toolbox-0'],
+ dependency_links=['git+https://github.com/apache/incubator-marvin@master#subdirectory=python-toolbox#egg=marvin_python_toolbox-0'],
tests_require=[
'pytest>=2.6.4',
'pytest-cov>=1.8.1',
diff --git a/public-engines/sample-engine/tests/conftest.py b/public-engines/sample-engine/tests/conftest.py
index 88fdff1..0817cc5 100644
--- a/public-engines/sample-engine/tests/conftest.py
+++ b/public-engines/sample-engine/tests/conftest.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/tests/data_handler/test_acquisitor_and_cleaner.py b/public-engines/sample-engine/tests/data_handler/test_acquisitor_and_cleaner.py
index 894f70c..0137c2a 100644
--- a/public-engines/sample-engine/tests/data_handler/test_acquisitor_and_cleaner.py
+++ b/public-engines/sample-engine/tests/data_handler/test_acquisitor_and_cleaner.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/tests/data_handler/test_training_preparator.py b/public-engines/sample-engine/tests/data_handler/test_training_preparator.py
index a547441..d7b13bf 100644
--- a/public-engines/sample-engine/tests/data_handler/test_training_preparator.py
+++ b/public-engines/sample-engine/tests/data_handler/test_training_preparator.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/tests/prediction/test_prediction_preparator.py b/public-engines/sample-engine/tests/prediction/test_prediction_preparator.py
index 500f2d1..8db7075 100644
--- a/public-engines/sample-engine/tests/prediction/test_prediction_preparator.py
+++ b/public-engines/sample-engine/tests/prediction/test_prediction_preparator.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/tests/prediction/test_predictor.py b/public-engines/sample-engine/tests/prediction/test_predictor.py
index ae0c10b..f83ed62 100644
--- a/public-engines/sample-engine/tests/prediction/test_predictor.py
+++ b/public-engines/sample-engine/tests/prediction/test_predictor.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/tests/training/test_metrics_evaluator.py b/public-engines/sample-engine/tests/training/test_metrics_evaluator.py
index a79a516..628ed7c 100644
--- a/public-engines/sample-engine/tests/training/test_metrics_evaluator.py
+++ b/public-engines/sample-engine/tests/training/test_metrics_evaluator.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sample-engine/tests/training/test_trainer.py b/public-engines/sample-engine/tests/training/test_trainer.py
index 1acfeef..1e5c6e5 100644
--- a/public-engines/sample-engine/tests/training/test_trainer.py
+++ b/public-engines/sample-engine/tests/training/test_trainer.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sms-spam-engine/Dockerfile b/public-engines/sms-spam-engine/Dockerfile
index 9ba1140..ed27aaf 100644
--- a/public-engines/sms-spam-engine/Dockerfile
+++ b/public-engines/sms-spam-engine/Dockerfile
@@ -1,6 +1,6 @@
FROM ubuntu:16.04
-MAINTAINER marvin-ai@googlegroups.com
+MAINTAINER dev@marvin.apache.org
ENV SLEEP_MILLIS 0
@@ -74,12 +74,6 @@
# spark configuration files to the think works correctly.
##############################################################
-# RUN curl https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz -o /tmp/spark-2.1.1-bin-hadoop2.6.tgz && \
-# tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/ && \
-# ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark
-
-# Add the b2w datalake config for Spark
-# ADD spark-conf.tar $SPARK_CONF_DIR
RUN mkdir -p $SPARK_CONF_DIR
##############################################################
diff --git a/public-engines/sms-spam-engine/docs.yaml b/public-engines/sms-spam-engine/docs.yaml
index 1470529..b9630fd 100644
--- a/public-engines/sms-spam-engine/docs.yaml
+++ b/public-engines/sms-spam-engine/docs.yaml
@@ -4,8 +4,8 @@
title: marvin_sms_spam_engine API Doc
contact:
name: mantainer Marvin AI Community
- email: marvin-ai@googlegroups.com
- url: https://github.com/marvin-ai
+ email: dev@marvin.apache.org
+ url: https://marvin.apache.org
license:
name: Apache License 2.0
servers:
diff --git a/public-engines/sms-spam-engine/marvin_sms_spam_engine/prediction/feedback.py b/public-engines/sms-spam-engine/marvin_sms_spam_engine/prediction/feedback.py
index d7a0ea6..c47bfff 100644
--- a/public-engines/sms-spam-engine/marvin_sms_spam_engine/prediction/feedback.py
+++ b/public-engines/sms-spam-engine/marvin_sms_spam_engine/prediction/feedback.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/public-engines/sms-spam-engine/setup.py b/public-engines/sms-spam-engine/setup.py
index a43196f..38dcbed 100644
--- a/public-engines/sms-spam-engine/setup.py
+++ b/public-engines/sms-spam-engine/setup.py
@@ -59,7 +59,7 @@
long_description=open(join(dirname(__file__), 'README.md')).read(),
author='Marvin AI Community',
maintainer='Marvin AI Community',
- maintainer_email='marvin-ai@googlegroups.com',
+ maintainer_email='dev@marvin.apache.org',
packages=find_packages(exclude=('tests', 'tests.*')),
include_package_data=True,
zip_safe=False,
@@ -84,7 +84,7 @@
'marvin-python-toolbox==0',
'Fabric==1.14.0',
],
- dependency_links=['git+https://github.com/marvin-ai/marvin-python-toolbox.git/@master#egg=marvin_python_toolbox-0'],
+ dependency_links=['git+https://github.com/apache/incubator-marvin@master#subdirectory=python-toolbox#egg=marvin_python_toolbox-0'],
tests_require=REQUIREMENT_TESTS,
extras_require={
'testing': REQUIREMENT_TESTS,
diff --git a/python-toolbox/CODE_OF_CONDUCT.md b/python-toolbox/CODE_OF_CONDUCT.md
index 72ec3a7..d1ea5fa 100644
--- a/python-toolbox/CODE_OF_CONDUCT.md
+++ b/python-toolbox/CODE_OF_CONDUCT.md
@@ -34,7 +34,7 @@
## Enforcement
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at marvin-ai@googlegroups.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at dev@marvin.apache.org. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
diff --git a/python-toolbox/Makefile b/python-toolbox/Makefile
index 060af38..d0417d4 100644
--- a/python-toolbox/Makefile
+++ b/python-toolbox/Makefile
@@ -1,4 +1,4 @@
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/NOTICE b/python-toolbox/NOTICE
index bf5c89e..45cc502 100644
--- a/python-toolbox/NOTICE
+++ b/python-toolbox/NOTICE
@@ -1,5 +1,5 @@
Marvin Artifical Intelligence Platform
-Copyright 2017 B2W Digital
+Copyright 2019 Apache Software Foundation
This product includes software developed at
-B2W Digital (http://www.b2w.digital)
\ No newline at end of file
+Apache Software Foundation https://www.apache.org/
\ No newline at end of file
diff --git a/python-toolbox/PULL_REQUEST_TEMPLATE.md b/python-toolbox/PULL_REQUEST_TEMPLATE.md
index 0334db7..82eda38 100644
--- a/python-toolbox/PULL_REQUEST_TEMPLATE.md
+++ b/python-toolbox/PULL_REQUEST_TEMPLATE.md
@@ -14,4 +14,4 @@
-
-
-@marvin-ai/marvin-core-team
+dev@marvin.apache.org
diff --git a/python-toolbox/README.md b/python-toolbox/README.md
index 6c18015..ef81a72 100644
--- a/python-toolbox/README.md
+++ b/python-toolbox/README.md
@@ -1,4 +1,4 @@
-[](https://travis-ci.org/marvin-ai/marvin-python-toolbox) [](https://codecov.io/gh/marvin-ai/marvin-python-toolbox)
+[](https://travis-ci.org/apache/incubator-marvin) [](https://codecov.io/gh/apache/incubator-marvin/branch/develop)
# Marvin Toolbox v0.0.5
@@ -92,7 +92,8 @@
1. Clone the example engine from the repository
```
-git clone https://github.com/marvin-ai/engines.git
+git clone https://github.com/apache/incubator-marvin.git
+cd public-engines
```
2. Generate a new Marvin engine environment for the Iris species engine
diff --git a/python-toolbox/bin/marvin b/python-toolbox/bin/marvin
index 80c2f6a..ac87ad5 100644
--- a/python-toolbox/bin/marvin
+++ b/python-toolbox/bin/marvin
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/__init__.py b/python-toolbox/marvin_python_toolbox/__init__.py
index cdde2f0..9ba50c3 100644
--- a/python-toolbox/marvin_python_toolbox/__init__.py
+++ b/python-toolbox/marvin_python_toolbox/__init__.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/_compatibility.py b/python-toolbox/marvin_python_toolbox/_compatibility.py
index c796f4e..0e4e321 100644
--- a/python-toolbox/marvin_python_toolbox/_compatibility.py
+++ b/python-toolbox/marvin_python_toolbox/_compatibility.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/_logging.py b/python-toolbox/marvin_python_toolbox/_logging.py
index 45635a8..ed999de 100644
--- a/python-toolbox/marvin_python_toolbox/_logging.py
+++ b/python-toolbox/marvin_python_toolbox/_logging.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/common/__init__.py b/python-toolbox/marvin_python_toolbox/common/__init__.py
index ba979d3..efb0a92 100644
--- a/python-toolbox/marvin_python_toolbox/common/__init__.py
+++ b/python-toolbox/marvin_python_toolbox/common/__init__.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/common/config.py b/python-toolbox/marvin_python_toolbox/common/config.py
index c91bfd5..9aa86c2 100644
--- a/python-toolbox/marvin_python_toolbox/common/config.py
+++ b/python-toolbox/marvin_python_toolbox/common/config.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/common/data.py b/python-toolbox/marvin_python_toolbox/common/data.py
index d86bd5b..12b905a 100644
--- a/python-toolbox/marvin_python_toolbox/common/data.py
+++ b/python-toolbox/marvin_python_toolbox/common/data.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/common/data_source_provider.py b/python-toolbox/marvin_python_toolbox/common/data_source_provider.py
index 2a2e78e..17f982b 100644
--- a/python-toolbox/marvin_python_toolbox/common/data_source_provider.py
+++ b/python-toolbox/marvin_python_toolbox/common/data_source_provider.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/common/exceptions.py b/python-toolbox/marvin_python_toolbox/common/exceptions.py
index 309f21e..3801298 100644
--- a/python-toolbox/marvin_python_toolbox/common/exceptions.py
+++ b/python-toolbox/marvin_python_toolbox/common/exceptions.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/common/http_client.py b/python-toolbox/marvin_python_toolbox/common/http_client.py
index a198c04..ad73f49 100644
--- a/python-toolbox/marvin_python_toolbox/common/http_client.py
+++ b/python-toolbox/marvin_python_toolbox/common/http_client.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/common/profiling.py b/python-toolbox/marvin_python_toolbox/common/profiling.py
index 1ed4e6b..5528427 100644
--- a/python-toolbox/marvin_python_toolbox/common/profiling.py
+++ b/python-toolbox/marvin_python_toolbox/common/profiling.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/common/utils.py b/python-toolbox/marvin_python_toolbox/common/utils.py
index 3799d61..9979f65 100644
--- a/python-toolbox/marvin_python_toolbox/common/utils.py
+++ b/python-toolbox/marvin_python_toolbox/common/utils.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/config.py b/python-toolbox/marvin_python_toolbox/config.py
index 99654c8..025ad3d 100644
--- a/python-toolbox/marvin_python_toolbox/config.py
+++ b/python-toolbox/marvin_python_toolbox/config.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/decorators.py b/python-toolbox/marvin_python_toolbox/decorators.py
index 976b4a2..fab885e 100644
--- a/python-toolbox/marvin_python_toolbox/decorators.py
+++ b/python-toolbox/marvin_python_toolbox/decorators.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/engine_base/__init__.py b/python-toolbox/marvin_python_toolbox/engine_base/__init__.py
index 2fc12f0..70dd9f9 100644
--- a/python-toolbox/marvin_python_toolbox/engine_base/__init__.py
+++ b/python-toolbox/marvin_python_toolbox/engine_base/__init__.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/engine_base/engine_base_action.py b/python-toolbox/marvin_python_toolbox/engine_base/engine_base_action.py
index fcabdb1..bb2eda1 100644
--- a/python-toolbox/marvin_python_toolbox/engine_base/engine_base_action.py
+++ b/python-toolbox/marvin_python_toolbox/engine_base/engine_base_action.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/engine_base/engine_base_data_handler.py b/python-toolbox/marvin_python_toolbox/engine_base/engine_base_data_handler.py
index df0a011..3c798fb 100644
--- a/python-toolbox/marvin_python_toolbox/engine_base/engine_base_data_handler.py
+++ b/python-toolbox/marvin_python_toolbox/engine_base/engine_base_data_handler.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/engine_base/engine_base_prediction.py b/python-toolbox/marvin_python_toolbox/engine_base/engine_base_prediction.py
index 1761c3b..af27228 100644
--- a/python-toolbox/marvin_python_toolbox/engine_base/engine_base_prediction.py
+++ b/python-toolbox/marvin_python_toolbox/engine_base/engine_base_prediction.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/engine_base/engine_base_training.py b/python-toolbox/marvin_python_toolbox/engine_base/engine_base_training.py
index 7df7d9f..5ff8e84 100644
--- a/python-toolbox/marvin_python_toolbox/engine_base/engine_base_training.py
+++ b/python-toolbox/marvin_python_toolbox/engine_base/engine_base_training.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/engine_base/protos/actions.proto b/python-toolbox/marvin_python_toolbox/engine_base/protos/actions.proto
index 4ab7f60..4a63daf 100644
--- a/python-toolbox/marvin_python_toolbox/engine_base/protos/actions.proto
+++ b/python-toolbox/marvin_python_toolbox/engine_base/protos/actions.proto
@@ -1,4 +1,4 @@
-/** Copyright [2017] [B2W Digital]
+/** Copyright [2019] [Apache Software Foundation]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/engine_base/serializers/__init__.py b/python-toolbox/marvin_python_toolbox/engine_base/serializers/__init__.py
index 73f7796..eef28de 100644
--- a/python-toolbox/marvin_python_toolbox/engine_base/serializers/__init__.py
+++ b/python-toolbox/marvin_python_toolbox/engine_base/serializers/__init__.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/engine_base/serializers/keras_serializer.py b/python-toolbox/marvin_python_toolbox/engine_base/serializers/keras_serializer.py
index 1407960..467d573 100644
--- a/python-toolbox/marvin_python_toolbox/engine_base/serializers/keras_serializer.py
+++ b/python-toolbox/marvin_python_toolbox/engine_base/serializers/keras_serializer.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/engine_base/stubs/__init__.py b/python-toolbox/marvin_python_toolbox/engine_base/stubs/__init__.py
index 613c8bc..1ede8e4 100644
--- a/python-toolbox/marvin_python_toolbox/engine_base/stubs/__init__.py
+++ b/python-toolbox/marvin_python_toolbox/engine_base/stubs/__init__.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/extras/notebook_extensions/jupyter_notebook_config.py b/python-toolbox/marvin_python_toolbox/extras/notebook_extensions/jupyter_notebook_config.py
index b5f7de0..5b44b9a 100644
--- a/python-toolbox/marvin_python_toolbox/extras/notebook_extensions/jupyter_notebook_config.py
+++ b/python-toolbox/marvin_python_toolbox/extras/notebook_extensions/jupyter_notebook_config.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/extras/notebook_extensions/main.js b/python-toolbox/marvin_python_toolbox/extras/notebook_extensions/main.js
index dea722b..efcc59b 100644
--- a/python-toolbox/marvin_python_toolbox/extras/notebook_extensions/main.js
+++ b/python-toolbox/marvin_python_toolbox/extras/notebook_extensions/main.js
@@ -1,4 +1,4 @@
-// Copyright [2017] [B2W Digital]
+// Copyright [2019] [Apache Software Foundation]
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/loader.py b/python-toolbox/marvin_python_toolbox/loader.py
index 7e54db3..e82be7a 100644
--- a/python-toolbox/marvin_python_toolbox/loader.py
+++ b/python-toolbox/marvin_python_toolbox/loader.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/manage.py b/python-toolbox/marvin_python_toolbox/manage.py
index 2fc817a..753794f 100644
--- a/python-toolbox/marvin_python_toolbox/manage.py
+++ b/python-toolbox/marvin_python_toolbox/manage.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/management/__init__.py b/python-toolbox/marvin_python_toolbox/management/__init__.py
index a4ce0ca..3b80667 100644
--- a/python-toolbox/marvin_python_toolbox/management/__init__.py
+++ b/python-toolbox/marvin_python_toolbox/management/__init__.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -28,6 +28,7 @@
from .notebook import cli as cli_notebook
from .hive import cli as cli_hive
from .engine import cli as cli_engine
+from .bigquery import cli as cli_bigquery
from ..config import parse_ini
from ..loader import load_commands_from_file
@@ -38,7 +39,7 @@
logger = get_logger('management')
-TOOL_EXCLUDE = ['engine-server', 'engine-dryrun', 'engine-httpserver', 'engine-grpcserver', 'engine-deploy', 'engine-httpserver-remote', 'pkg-showversion']
+TOOL_EXCLUDE = ['engine-server', 'engine-dryrun', 'engine-httpserver', 'engine-grpcserver', 'engine-deploy', 'engine-httpserver-remote', 'pkg-showversion', 'bigquery-dataimport']
PROD_EXCLUDE = ['test', 'test-tdd', 'test-tox', 'test-checkpep8', 'lab', 'notebook', 'pkg-bumpversion', 'pkg-createtag', 'pkg-showchanges', 'pkg-showinfo', 'pkg-updatedeps']
EXCLUDE_BY_TYPE = {
@@ -108,6 +109,7 @@
# Load internal commands
commands = {}
+ commands.update(cli_bigquery.commands)
commands.update(cli_pkg.commands)
commands.update(cli_test.commands)
commands.update(cli_notebook.commands)
diff --git a/python-toolbox/marvin_python_toolbox/management/bigquery.py b/python-toolbox/marvin_python_toolbox/management/bigquery.py
new file mode 100644
index 0000000..b18a7b8
--- /dev/null
+++ b/python-toolbox/marvin_python_toolbox/management/bigquery.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2019] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import click
+import time
+import os
+import json
+import pandas as pd
+from google.cloud import bigquery
+import hashlib
+
+from .._logging import get_logger
+
+from .._compatibility import six
+
+
+logger = get_logger('management.bigquery')
+
+
+@click.group('bigquery')
+def cli():
+ pass
+
+def read_file(filename):
+ fname = os.path.join("", filename)
+ if os.path.exists(fname):
+
+ print("Engine file {} loaded!".format(filename))
+
+ with open(fname, 'r') as fp:
+ return json.load(fp)
+ else:
+ print("Engine file {} doesn't exists...".format(filename))
+ return {}
+
+@cli.command(
+ 'bigquery-dataimport',
+ help='Import data samples from a BigQuery database.')
+@click.option('--dryrun', '-d', is_flag=True, help='If it must run just to estimate costs')
+@click.option('--max_billed', '-m', default=10, help='Max bytes to be billed in the queries')
+@click.option('--metadata-file', '-mf', default='engine.metadata', help='Marvin engine metadata file path', type=click.Path(exists=True))
+@click.pass_context
+def bigquery_dataimport_cli(ctx, dryrun, max_billed, metadata_file):
+ bigquery_dataimport(ctx, metadata_file, dryrun, max_billed)
+
+def bigquery_dataimport(ctx, metadata_file, dryrun, max_billed):
+
+ initial_start_time = time.time()
+
+ metadata = read_file(metadata_file)
+
+ if metadata:
+ print(chr(27) + "[2J")
+
+ data_path = os.environ['MARVIN_DATA_PATH']
+ path_csvs = data_path + '/bigquery-' + metadata['bigquery_project']
+
+ if not dryrun:
+ os.mkdir(path_csvs)
+
+ for query, file in zip(metadata['bigquery_queries'], metadata['bigquery_csvfiles']):
+
+ print("project: {} query: {} file: {}".format(metadata['bigquery_project'], query, file))
+
+ bdi = BigQueryImporter(
+ project=metadata['bigquery_project'],
+ sql=query,
+ file=file,
+ # transform max_billed in bytes
+ max_billed=max_billed * 1073741824,
+ path_csv = path_csvs
+
+ )
+ if dryrun:
+ bdi.dryrun()
+ else:
+ bdi.query()
+
+ print("Total Time : {:.2f}s".format(time.time() - initial_start_time))
+
+ print("\n")
+
+
+def read_config(filename):
+ fname = os.path.join("", filename)
+ if os.path.exists(fname):
+ with open(fname, 'r') as fp:
+ return json.load(fp)[0]
+ else:
+ print("Configuration file {} doesn't exists...".format(filename))
+ return {}
+
+
+class BigQueryImporter():
+ def __init__(self, project, sql, file, max_billed, path_csv):
+ self.project = project
+ self.sql = sql
+ self.file = file
+ self.max_billed = max_billed
+ self.path_csv = path_csv
+
+ def query(self):
+ job_config = bigquery.QueryJobConfig()
+ job_config.use_query_cache = False
+ job_config.maximum_bytes_billed=self.max_billed
+ client = bigquery.Client(project=self.project)
+ query_job = client.query(self.sql,
+ job_config=job_config)
+ dataframe = query_job.to_dataframe()
+ dataframe.to_csv(self.path_csv + '/' + self.file, index=False)
+
+ def dryrun(self):
+ job_config = bigquery.QueryJobConfig()
+ job_config.use_query_cache = False
+ job_config.dry_run = True
+ job_config.maximum_bytes_billed=self.max_billed
+ client = bigquery.Client(project=self.project)
+ query_job = client.query(self.sql,
+ job_config=job_config)
+
+ assert query_job.state == "DONE"
+ assert query_job.dry_run
+
+ print("The query: {}\nWill process {} Gb.\n".format(self.sql, query_job.total_bytes_processed / 1073741824))
diff --git a/python-toolbox/marvin_python_toolbox/management/engine.py b/python-toolbox/marvin_python_toolbox/management/engine.py
index f15dbeb..2c80940 100644
--- a/python-toolbox/marvin_python_toolbox/management/engine.py
+++ b/python-toolbox/marvin_python_toolbox/management/engine.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -32,7 +32,7 @@
import multiprocessing
from marvin_python_toolbox.common.profiling import profiling
from marvin_python_toolbox.common.data import MarvinData
-from marvin_python_toolbox.common.config import Config
+from marvin_python_toolbox.common.config import Config, load_conf_from_file
from .._compatibility import iteritems
from .._logging import get_logger
@@ -336,7 +336,8 @@
TEMPLATE_BASES = {
- 'python-engine': os.path.join(os.path.dirname(__file__), 'templates', 'python-engine')
+ 'python-engine': os.path.join(os.path.dirname(__file__), 'templates', 'python-engine'),
+ 'automl-engine': os.path.join(os.path.dirname(__file__), 'templates', 'python-engine'),
}
RENAME_DIRS = [
@@ -356,9 +357,10 @@
@click.argument('engine-path', type=click.Path(exists=True))
@click.option('--python', '-p', default='python', help='The Python interpreter to use to create the new environment')
def generate_env(engine_path, python):
+ engine_type = load_conf_from_file(engine_path + '/marvin.ini').get('type')
dir_ = os.path.basename(os.path.abspath(engine_path))
venv_name = _create_virtual_env(dir_, engine_path, python)
- _call_make_env(venv_name)
+ _call_make_env(venv_name, engine_type)
print('\nDone!!!!')
print('Now to workon in the new engine project use: workon {}'.format(venv_name))
@@ -368,18 +370,23 @@
@click.option('--name', '-n', prompt='Project name', help='Project name')
@click.option('--description', '-d', prompt='Short description', default='Marvin engine', help='Library short description')
@click.option('--mantainer', '-m', prompt='Mantainer name', default='Marvin AI Community', help='Mantainer name')
-@click.option('--email', '-e', prompt='Mantainer email', default='marvin-ai@googlegroups.com', help='Mantainer email')
+@click.option('--email', '-e', prompt='Mantainer email', default='dev@marvin.apache.org', help='Mantainer email')
@click.option('--package', '-p', default='', help='Package name')
@click.option('--dest', '-d', envvar='MARVIN_HOME', type=click.Path(exists=True), help='Root folder path for the creation')
@click.option('--no-env', is_flag=True, default=False, help='Don\'t create the virtual enviroment')
@click.option('--no-git', is_flag=True, default=False, help='Don\'t initialize the git repository')
+@click.option('--automl', '-aml', default='n' ,prompt='Use AutoML?: ', type=click.Choice(['y','n']))
@click.option('--python', '-py', default='python', help='The Python interpreter to use to create the new environment')
-def generate(name, description, mantainer, email, package, dest, no_env, no_git, python):
+def generate(name, description, mantainer, email, package, dest, no_env, no_git, automl, python):
type_ = 'python-engine'
type = _orig_type
- # Process package name
+ # Check if package should be automl
+ if automl == 'y':
+ type_ = 'automl-engine'
+
+ # Process package name
package = _slugify(package or name)
# Make sure package name starts with "marvin"
@@ -394,7 +401,7 @@
# Append project type to services
- if type_ == 'python-engine' and not package.endswith('engine'):
+ if type_ in TEMPLATE_BASES and not package.endswith('engine'):
package = '{}_engine'.format(package)
# Process directory/virtualenv name
@@ -449,7 +456,7 @@
venv_name = None
if not no_env:
venv_name = _create_virtual_env(dir_, dest, python)
- _call_make_env(venv_name)
+ _call_make_env(venv_name, type_)
if not no_git:
_call_git_init(dest)
@@ -461,6 +468,7 @@
except Exception as e:
logger.info(e)
+ print("\nAlready exists a engine project with this name!")
# remove project if created
if os.path.exists(dest) and folder_created:
shutil.rmtree(dest)
@@ -552,8 +560,11 @@
return venv_name
-def _call_make_env(venv_name):
- command = ['bash', '-c', '. virtualenvwrapper.sh; workon {}; make marvin'.format(venv_name)]
+def _call_make_env(venv_name, engine):
+ if engine == 'automl-engine':
+ command = ['bash', '-c', '. virtualenvwrapper.sh; workon {}; make marvin-automl'.format(venv_name)]
+ else:
+ command = ['bash', '-c', '. virtualenvwrapper.sh; workon {}; make marvin'.format(venv_name)]
try:
subprocess.Popen(command, env=os.environ).wait()
diff --git a/python-toolbox/marvin_python_toolbox/management/hive.py b/python-toolbox/marvin_python_toolbox/management/hive.py
index b137846..3d828b2 100644
--- a/python-toolbox/marvin_python_toolbox/management/hive.py
+++ b/python-toolbox/marvin_python_toolbox/management/hive.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/management/notebook.py b/python-toolbox/marvin_python_toolbox/management/notebook.py
index eb4e761..3e748d2 100644
--- a/python-toolbox/marvin_python_toolbox/management/notebook.py
+++ b/python-toolbox/marvin_python_toolbox/management/notebook.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/management/pkg.py b/python-toolbox/marvin_python_toolbox/management/pkg.py
index d126f68..a283390 100644
--- a/python-toolbox/marvin_python_toolbox/management/pkg.py
+++ b/python-toolbox/marvin_python_toolbox/management/pkg.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/management/templates/python-engine/Dockerfile b/python-toolbox/marvin_python_toolbox/management/templates/python-engine/Dockerfile
index 08deb81..ec237ff 100644
--- a/python-toolbox/marvin_python_toolbox/management/templates/python-engine/Dockerfile
+++ b/python-toolbox/marvin_python_toolbox/management/templates/python-engine/Dockerfile
@@ -1,4 +1,9 @@
-FROM ubuntu:16.04
+############################################################
+FROM python:2-alpine3.10
+# To use python3 comment the line above and uncomment
+# the line bellow.
+#FROM python:3-alpine3.10
+############################################################
MAINTAINER {{mantainer.email}}
@@ -21,6 +26,7 @@
ENV YARN_CONF_DIR=$SPARK_CONF_DIR
+
##############################################################
# Create all folders needed
##############################################################
@@ -29,77 +35,50 @@
mkdir -p $MARVIN_DATA_PATH && \
mkdir -p $MARVIN_ENGINE_HOME && \
mkdir -p /var/log/marvin/engines && \
- mkdir -p /var/run/marvin/engines
-
-
+ mkdir -p /var/run/marvin/engines && \
##############################################################
# Install the system dependencies for default installation
##############################################################
-
-RUN apt-get update -y && \
- apt-get install -y build-essential && \
- apt-get install -y maven git python cmake software-properties-common curl libstdc++6 && \
- apt-get install -y git && \
- apt-get install -y wget && \
- apt-get install -y python2.7-dev && \
- apt-get install -y python-pip && \
- apt-get install -y ipython && \
- apt-get install -y libffi-dev && \
- apt-get install -y libssl-dev && \
- apt-get install -y libxml2-dev && \
- apt-get install -y libxslt1-dev && \
- apt-get install -y libpng12-dev && \
- apt-get install -y libfreetype6-dev && \
- apt-get install -y python-tk && \
- apt-get install -y libsasl2-dev && \
- apt-get install -y python-pip && \
- apt-get install -y graphviz && \
- pip install --upgrade pip && \
- apt-get clean
-
-RUN pip install virtualenvwrapper
-
-# Install Oracle JDK
-RUN add-apt-repository ppa:webupd8team/java -y && \
- apt-get -qq update && \
- echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \
- echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \
- apt-get install -y oracle-java8-installer
-
-
+ apk add --no-cache g++ openssl-dev openjdk11-jre-headless bash && \
+ apk add --no-cache --virtual .build-deps make \
+ git \
+ wget \
+ libsass-dev \
+ openblas-dev \
+ libffi-dev \
+ libxml2-dev \
+ libxslt-dev \
+ libpng-dev \
+ freetype-dev \
+ cyrus-sasl-dev
##############################################################
# Install Apache Spark
#
# Uncomment if you are using spark, note that is needed the
# spark configuration files to the think works correctly.
##############################################################
-
-# RUN curl https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz -o /tmp/spark-2.1.1-bin-hadoop2.6.tgz && \
+#
+# RUN wget -O /tmp/spark-2.1.1-bin-hadoop2.6.tgz https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz && \
# tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/ && \
# ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark
+##############################################################
-# Add the b2w datalake config for Spark
-# ADD spark-conf.tar $SPARK_CONF_DIR
RUN mkdir -p $SPARK_CONF_DIR
##############################################################
-# Create the virtualenv configuration
-##############################################################
-
-RUN /bin/bash -c "cd $MARVIN_ENGINE_HOME && \
- source /usr/local/bin/virtualenvwrapper.sh && \
- mkvirtualenv $MARVIN_ENGINE_ENV"
-
-
-##############################################################
# <CUSTOM ENGINE INSTALLATION PROCEDURE HERE> #
##############################################################
##############################################################
-# Copy and Install the marvin engine inside virtualenv
+# Copy and Install the marvin engine
##############################################################
+RUN /bin/bash -c "pip install virtualenvwrapper && \
+ cd $MARVIN_ENGINE_HOME && \
+ source /usr/local/bin/virtualenvwrapper.sh && \
+ mkvirtualenv $MARVIN_ENGINE_ENV"
+
ADD build/engine.tar $MARVIN_ENGINE_HOME
ADD build/marvin-engine-executor-assembly.jar $MARVIN_DATA_PATH
@@ -107,8 +86,20 @@
RUN /bin/bash -c "source /usr/local/bin/virtualenvwrapper.sh && \
workon $MARVIN_ENGINE_ENV && \
cd $MARVIN_ENGINE_HOME && \
- pip install . --process-dependency-links"
-
+ pip install --no-cache numpy && \
+ pip install --no-cache scipy && \
+ pip install --no-cache pandas && \
+ pip install --no-cache matplotlib && \
+ pip install --no-cache cython && \
+ pip install --no-cache scikit-learn && \
+ pip install --no-cache Fabric && \
+ pip install --no-cache marvin-python-toolbox && \
+ pip install . "
+##############################################################
+# Uninstalling unnecessary software and cleaning cache
+##############################################################
+RUN rm -rf /root/.cache && \
+ apk del .build-deps
##############################################################
# Starts the engine http server
@@ -119,4 +110,4 @@
CMD /bin/bash -c "source /usr/local/bin/virtualenvwrapper.sh && \
workon $MARVIN_ENGINE_ENV && \
cd $MARVIN_ENGINE_HOME && \
- marvin engine-httpserver -h 0.0.0.0 -p 8000"
\ No newline at end of file
+ marvin engine-httpserver -h 0.0.0.0 -p 8000"
diff --git a/python-toolbox/marvin_python_toolbox/management/templates/python-engine/Makefile b/python-toolbox/marvin_python_toolbox/management/templates/python-engine/Makefile
index e734143..59032ed 100644
--- a/python-toolbox/marvin_python_toolbox/management/templates/python-engine/Makefile
+++ b/python-toolbox/marvin_python_toolbox/management/templates/python-engine/Makefile
@@ -1,4 +1,4 @@
-.PHONY: help marvin marvin-prod update clean-pyc clean-build clean-reports clean-deps clean docker-build docker-push docker-run
+.PHONY: help marvin marvin-automl marvin-prod update clean-pyc clean-build clean-reports clean-deps clean docker-build docker-push docker-run
DOCKER_VERSION?=0.00.01
DOCKER_REGISTRY_ADRESS?=docker.registry.io
@@ -11,6 +11,8 @@
@echo " Prepare project to be used as a marvin package."
@echo " marvin-prod"
@echo " Prepare project to be used in production environment."
+ @echo " marvin-automl"
+ @echo " Prepare project to be used as a marvin-automl package."
@echo " update"
@echo " Reinstall requirements and setup.py dependencies."
@echo " clean"
@@ -33,7 +35,10 @@
marvin:
pip install -e ".[testing]"
marvin --help
-
+marvin-automl:
+ pip install -e ".[testing]"
+ bash install_automl.sh
+ marvin --help
marvin-prod:
pip install .
marvin --help
@@ -73,4 +78,4 @@
sudo docker run --name=marvin-$(MARVIN_ENGINE_NAME)-$(DOCKER_VERSION) --mount type=bind,source=$(MARVIN_DATA_PATH),destination=/marvin-data -p 8000:8000 $(DOCKER_REGISTRY_ADRESS)/$(MARVIN_ENGINE_NAME):$(DOCKER_VERSION)
docker-push:
- sudo docker push $(DOCKER_REGISTRY_ADRESS)/$(MARVIN_ENGINE_NAME):$(DOCKER_VERSION)
\ No newline at end of file
+ sudo docker push $(DOCKER_REGISTRY_ADRESS)/$(MARVIN_ENGINE_NAME):$(DOCKER_VERSION)
diff --git a/python-toolbox/marvin_python_toolbox/management/templates/python-engine/docs.yaml b/python-toolbox/marvin_python_toolbox/management/templates/python-engine/docs.yaml
index 6983f8d..d701870 100644
--- a/python-toolbox/marvin_python_toolbox/management/templates/python-engine/docs.yaml
+++ b/python-toolbox/marvin_python_toolbox/management/templates/python-engine/docs.yaml
@@ -5,7 +5,7 @@
contact:
name: mantainer {{mantainer.name}}
email: {{mantainer.email}}
- url: https://github.com/marvin-ai
+ url: https://marvin.apache.org
license:
name: Apache License 2.0
servers:
diff --git a/python-toolbox/marvin_python_toolbox/management/templates/python-engine/engine.metadata b/python-toolbox/marvin_python_toolbox/management/templates/python-engine/engine.metadata
index 88ecc2f..bb56837 100644
--- a/python-toolbox/marvin_python_toolbox/management/templates/python-engine/engine.metadata
+++ b/python-toolbox/marvin_python_toolbox/management/templates/python-engine/engine.metadata
@@ -58,5 +58,8 @@
"artifactsToPersist": [],
"artifactsToLoad": [],
"pipeline": []
- }]
+ }],
+ "bigquery_project": "xxx-project",
+ "bigquery_queries": [],
+ "bigquery_csvfiles": []
}
diff --git a/python-toolbox/marvin_python_toolbox/management/templates/python-engine/install_automl.sh b/python-toolbox/marvin_python_toolbox/management/templates/python-engine/install_automl.sh
new file mode 100644
index 0000000..4bde530
--- /dev/null
+++ b/python-toolbox/marvin_python_toolbox/management/templates/python-engine/install_automl.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+###############################################
+# Marvin AutoML installation Script #
+###############################################
+cmd=(dialog --title "Marvin AutoML" --separate-output --checklist "Select tools:" 22 76 16)
+options=(1 "auto-sklearn" off # any option can be set to default to "on"
+ 2 "h2o AutoML" off
+ 3 "TPOT" off)
+choices=$("${cmd[@]}" "${options[@]}" 2>&1 >/dev/tty)
+clear
+for choice in $choices
+do
+ case $choice in
+ 1)
+ echo "Instaling auto-sklearn..."
+ wget https://raw.githubusercontent.com/automl/auto-sklearn/master/requirements.txt \
+ | xargs -n 1 -L 1 pip install
+ pip install auto-sklearn
+ ;;
+ 2)
+ echo "Installing h2o..."
+ pip install requests
+ pip install tabulate
+ pip install scikit-learn
+ pip install http://h2o-release.s3.amazonaws.com/h2o/rel-yau/3/Python/h2o-3.26.0.3-py2.py3-none-any.whl
+ wget http://h2o-release.s3.amazonaws.com/h2o/rel-yau/3/h2o-3.26.0.3.zip
+ unzip h2o-3.26.0.3.zip
+ rm h2o-3.26.0.3.zip
+ ;;
+ 3)
+ echo "Installing TPOT..."
+ pip install tpot
+ ;;
+ esac
+done
diff --git a/python-toolbox/marvin_python_toolbox/management/templates/python-engine/project_package/prediction/feedback.py b/python-toolbox/marvin_python_toolbox/management/templates/python-engine/project_package/prediction/feedback.py
index d7a0ea6..c47bfff 100644
--- a/python-toolbox/marvin_python_toolbox/management/templates/python-engine/project_package/prediction/feedback.py
+++ b/python-toolbox/marvin_python_toolbox/management/templates/python-engine/project_package/prediction/feedback.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/marvin_python_toolbox/management/test.py b/python-toolbox/marvin_python_toolbox/management/test.py
index 741e4af..0f837a5 100644
--- a/python-toolbox/marvin_python_toolbox/management/test.py
+++ b/python-toolbox/marvin_python_toolbox/management/test.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/setup.py b/python-toolbox/setup.py
index cfec315..b3a2553 100644
--- a/python-toolbox/setup.py
+++ b/python-toolbox/setup.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -30,7 +30,7 @@
URL = 'https://github.com/apache/incubator-marvin'
AUTHOR_NAME = 'Daniel Takabayashi'
-AUTHOR_EMAIL = 'taka@apache.org'
+AUTHOR_EMAIL = 'dev@marvin.apache.org'
PYTHON_2 = True
PYTHON_3 = True
@@ -46,7 +46,7 @@
# External dependencies
# More info https://pythonhosted.org/setuptools/setuptools.html#declaring-dependencies
REQUIREMENTS_EXTERNAL = [
- 'six>=1.10.0',
+ 'six>=1.14.0',
'bumpversion>=0.5.3',
'click>=3.3',
'jupyter>=1.0.0',
@@ -77,11 +77,13 @@
'joblib>=0.11',
'autopep8>=1.3.3',
'progressbar2>=3.34.3',
- 'urllib3>=1.21.1',
+ 'urllib3==1.21.1',
+ 'unidecode==1.0.23',
'idna>=2.5',
'bleach>=1.5.0',
'numpy>=1.16.2',
- 'Unidecode==1.0.23',
+ 'google-cloud-bigquery>=1.21',
+ 'pandas>=0.24'
]
# Test dependencies
@@ -91,9 +93,9 @@
'pytest>=2.9.2',
'pytest-cov>=1.8.1,<2.6',
'pytest-watch>=4.1.0',
- 'pytest-testmon>=0.8.2',
+ 'pytest-testmon==0.8.2',
'Keras>=2.2.0',
- 'tensorflow>=1.8.0',
+ 'tensorflow==2.0',
]
# This is normally an empty list
DEPENDENCY_LINKS_EXTERNAL = []
diff --git a/python-toolbox/tests/common/test_config.py b/python-toolbox/tests/common/test_config.py
index 6cdcda5..5baf001 100644
--- a/python-toolbox/tests/common/test_config.py
+++ b/python-toolbox/tests/common/test_config.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/common/test_data.py b/python-toolbox/tests/common/test_data.py
index c244e81..99b34ef 100644
--- a/python-toolbox/tests/common/test_data.py
+++ b/python-toolbox/tests/common/test_data.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/common/test_data_source_provider.py b/python-toolbox/tests/common/test_data_source_provider.py
index ff5bce7..08a7633 100644
--- a/python-toolbox/tests/common/test_data_source_provider.py
+++ b/python-toolbox/tests/common/test_data_source_provider.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/common/test_http_client.py b/python-toolbox/tests/common/test_http_client.py
index 2b92844..10971eb 100644
--- a/python-toolbox/tests/common/test_http_client.py
+++ b/python-toolbox/tests/common/test_http_client.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/common/test_profiling.py b/python-toolbox/tests/common/test_profiling.py
index 62088ab..9ff5fc1 100644
--- a/python-toolbox/tests/common/test_profiling.py
+++ b/python-toolbox/tests/common/test_profiling.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/common/test_utils.py b/python-toolbox/tests/common/test_utils.py
index 5845811..2bd0af4 100644
--- a/python-toolbox/tests/common/test_utils.py
+++ b/python-toolbox/tests/common/test_utils.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/conftest.py b/python-toolbox/tests/conftest.py
index 30a578a..ebed059 100644
--- a/python-toolbox/tests/conftest.py
+++ b/python-toolbox/tests/conftest.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/engine_base/serializers/test_keras_serializer.py b/python-toolbox/tests/engine_base/serializers/test_keras_serializer.py
index 7cd42cc..4b53983 100644
--- a/python-toolbox/tests/engine_base/serializers/test_keras_serializer.py
+++ b/python-toolbox/tests/engine_base/serializers/test_keras_serializer.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/engine_base/test_engine_base_action.py b/python-toolbox/tests/engine_base/test_engine_base_action.py
index 3ac5647..b4d2010 100644
--- a/python-toolbox/tests/engine_base/test_engine_base_action.py
+++ b/python-toolbox/tests/engine_base/test_engine_base_action.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/engine_base/test_engine_base_data_handler.py b/python-toolbox/tests/engine_base/test_engine_base_data_handler.py
index e89e8cb..03c1648 100644
--- a/python-toolbox/tests/engine_base/test_engine_base_data_handler.py
+++ b/python-toolbox/tests/engine_base/test_engine_base_data_handler.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/engine_base/test_engine_base_prediction.py b/python-toolbox/tests/engine_base/test_engine_base_prediction.py
index 4e3c3fe..2a9d5e0 100644
--- a/python-toolbox/tests/engine_base/test_engine_base_prediction.py
+++ b/python-toolbox/tests/engine_base/test_engine_base_prediction.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/engine_base/test_engine_base_training.py b/python-toolbox/tests/engine_base/test_engine_base_training.py
index f809b57..6102b14 100644
--- a/python-toolbox/tests/engine_base/test_engine_base_training.py
+++ b/python-toolbox/tests/engine_base/test_engine_base_training.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/management/test_engine.py b/python-toolbox/tests/management/test_engine.py
index 91cb8ed..9daa4f4 100644
--- a/python-toolbox/tests/management/test_engine.py
+++ b/python-toolbox/tests/management/test_engine.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/management/test_hive.py b/python-toolbox/tests/management/test_hive.py
index a24e417..9d7ee7b 100644
--- a/python-toolbox/tests/management/test_hive.py
+++ b/python-toolbox/tests/management/test_hive.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/management/test_notebook.py b/python-toolbox/tests/management/test_notebook.py
index 2a31e0d..0c66eb7 100644
--- a/python-toolbox/tests/management/test_notebook.py
+++ b/python-toolbox/tests/management/test_notebook.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/management/test_pkg.py b/python-toolbox/tests/management/test_pkg.py
index de81226..2369967 100644
--- a/python-toolbox/tests/management/test_pkg.py
+++ b/python-toolbox/tests/management/test_pkg.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
diff --git a/python-toolbox/tests/test_loader.py b/python-toolbox/tests/test_loader.py
index 1cdc39e..d09ef0d 100644
--- a/python-toolbox/tests/test_loader.py
+++ b/python-toolbox/tests/test_loader.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# coding=utf-8
-# Copyright [2017] [B2W Digital]
+# Copyright [2019] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
