Add iris automl public engine
diff --git a/public-engines/iris-automl/.bumpversion.cfg b/public-engines/iris-automl/.bumpversion.cfg
new file mode 100644
index 0000000..102f3a3
--- /dev/null
+++ b/public-engines/iris-automl/.bumpversion.cfg
@@ -0,0 +1,5 @@
+[bumpversion]
+current_version = 0.0.1
+
+[bumpversion:file:marvin_iris_automl/VERSION]
+[bumpversion:file:README.md]
\ No newline at end of file
diff --git a/public-engines/iris-automl/.coveragerc b/public-engines/iris-automl/.coveragerc
new file mode 100644
index 0000000..6ebe78e
--- /dev/null
+++ b/public-engines/iris-automl/.coveragerc
@@ -0,0 +1,22 @@
+[run]
+omit = tests/*
+branch = True
+
+[report]
+exclude_lines =
+ pragma: no cover
+
+ def __repr__
+ if self\.debug
+
+ raise AssertionError
+ raise NotImplementedError
+
+ if 0:
+ if __name__ == .__main__.:
+
+[html]
+directory = coverage_report
+
+[xml]
+output = coverage_report.xml
\ No newline at end of file
diff --git a/public-engines/iris-automl/.gitignore b/public-engines/iris-automl/.gitignore
new file mode 100644
index 0000000..05dd2a2
--- /dev/null
+++ b/public-engines/iris-automl/.gitignore
@@ -0,0 +1,16 @@
+.cache
+.eggs
+.tox
+.testmondata
+.coverage
+.coverage.*
+coverage_report.xml
+coverage_report
+*.egg
+*.egg-info
+*.pyc
+tests/__pycache__
+.DS_Store
+.packages
+.profiling
+notebooks/data
\ No newline at end of file
diff --git a/public-engines/iris-automl/CHANGES.md b/public-engines/iris-automl/CHANGES.md
new file mode 100644
index 0000000..4835f6f
--- /dev/null
+++ b/public-engines/iris-automl/CHANGES.md
@@ -0,0 +1,5 @@
+## Changes log
+
+### 0.0.1
+
+ - initial version
\ No newline at end of file
diff --git a/public-engines/iris-automl/Dockerfile b/public-engines/iris-automl/Dockerfile
new file mode 100644
index 0000000..f117241
--- /dev/null
+++ b/public-engines/iris-automl/Dockerfile
@@ -0,0 +1,113 @@
+############################################################
+FROM python:2-alpine3.10
+# To use python3 comment the line above and uncomment
+# the line bellow.
+#FROM python:3-alpine3.10
+############################################################
+
+MAINTAINER dev@marvin.apache.org
+
+ENV SLEEP_MILLIS 0
+
+USER root
+
+##############################################################
+# Define all environment variables to be used
+##############################################################
+
+ENV MARVIN_HOME=/opt/marvin
+ENV MARVIN_DATA_PATH=/marvin-data
+ENV MARVIN_ENGINE_HOME=$MARVIN_HOME/engine
+ENV MARVIN_ENGINE_ENV=marvin-engine-env
+ENV WORKON_HOME=$MARVIN_HOME/.virtualenvs
+ENV SPARK_HOME=/opt/spark
+ENV SPARK_CONF_DIR=$SPARK_HOME/conf
+ENV HADOOP_CONF_DIR=$SPARK_CONF_DIR
+ENV YARN_CONF_DIR=$SPARK_CONF_DIR
+
+
+
+##############################################################
+# Create all folders needed
+##############################################################
+
+RUN mkdir -p $MARVIN_HOME && \
+ mkdir -p $MARVIN_DATA_PATH && \
+ mkdir -p $MARVIN_ENGINE_HOME && \
+ mkdir -p /var/log/marvin/engines && \
+ mkdir -p /var/run/marvin/engines && \
+##############################################################
+# Install the system dependencies for default installation
+##############################################################
+ apk add --no-cache g++ openssl-dev openjdk11-jre-headless bash && \
+ apk add --no-cache --virtual .build-deps make \
+ git \
+ wget \
+ libsass-dev \
+ openblas-dev \
+ libffi-dev \
+ libxml2-dev \
+ libxslt-dev \
+ libpng-dev \
+ freetype-dev \
+ cyrus-sasl-dev
+##############################################################
+# Install Apache Spark
+#
+# Uncomment if you are using spark, note that is needed the
+# spark configuration files to the think works correctly.
+##############################################################
+#
+# RUN wget -O /tmp/spark-2.1.1-bin-hadoop2.6.tgz https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz && \
+# tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/ && \
+# ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark
+##############################################################
+
+RUN mkdir -p $SPARK_CONF_DIR
+
+##############################################################
+# <CUSTOM ENGINE INSTALLATION PROCEDURE HERE> #
+##############################################################
+
+
+##############################################################
+# Copy and Install the marvin engine
+##############################################################
+
+RUN /bin/bash -c "pip install virtualenvwrapper && \
+ cd $MARVIN_ENGINE_HOME && \
+ source /usr/local/bin/virtualenvwrapper.sh && \
+ mkvirtualenv $MARVIN_ENGINE_ENV"
+
+ADD build/engine.tar $MARVIN_ENGINE_HOME
+
+ADD build/marvin-engine-executor-assembly.jar $MARVIN_DATA_PATH
+
+RUN /bin/bash -c "source /usr/local/bin/virtualenvwrapper.sh && \
+ workon $MARVIN_ENGINE_ENV && \
+ cd $MARVIN_ENGINE_HOME && \
+ pip install --no-cache numpy && \
+ pip install --no-cache scipy && \
+ pip install --no-cache pandas && \
+ pip install --no-cache matplotlib && \
+ pip install --no-cache cython && \
+ pip install --no-cache scikit-learn && \
+ pip install --no-cache Fabric && \
+ pip install --no-cache marvin-python-toolbox && \
+ pip install . "
+##############################################################
+# Uninstalling unnecessary software and cleaning cache
+##############################################################
+RUN rm -rf /root/.cache && \
+ apk del .build-deps
+
+##############################################################
+# Starts the engine http server
+##############################################################
+
+EXPOSE 8000
+
+CMD /bin/bash -c "source /usr/local/bin/virtualenvwrapper.sh && \
+ workon $MARVIN_ENGINE_ENV && \
+ cd $MARVIN_ENGINE_HOME && \
+ marvin engine-httpserver -h 0.0.0.0 -p 8000"
\ No newline at end of file
diff --git a/public-engines/iris-automl/INSTALL b/public-engines/iris-automl/INSTALL
new file mode 100644
index 0000000..fccdaf8
--- /dev/null
+++ b/public-engines/iris-automl/INSTALL
@@ -0,0 +1 @@
+REPLACE: Add here the detailed instructions to install this project
\ No newline at end of file
diff --git a/public-engines/iris-automl/LICENSE b/public-engines/iris-automl/LICENSE
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/public-engines/iris-automl/LICENSE
diff --git a/public-engines/iris-automl/MANIFEST.in b/public-engines/iris-automl/MANIFEST.in
new file mode 100644
index 0000000..2d283ba
--- /dev/null
+++ b/public-engines/iris-automl/MANIFEST.in
@@ -0,0 +1,9 @@
+include CHANGES.md
+include INSTALL
+include LICENSE
+include MANIFEST.in
+include README.md
+include marvin_iris_automl/VERSION
+recursive-include notebooks *
+prune notebooks/build
+recursive-include tests *
\ No newline at end of file
diff --git a/public-engines/iris-automl/Makefile b/public-engines/iris-automl/Makefile
new file mode 100644
index 0000000..a1a8399
--- /dev/null
+++ b/public-engines/iris-automl/Makefile
@@ -0,0 +1,81 @@
+.PHONY: help marvin marvin-automl marvin-prod update clean-pyc clean-build clean-reports clean-deps clean docker-build docker-push docker-run
+
+DOCKER_VERSION?=0.00.01
+DOCKER_REGISTRY_ADRESS?=docker.registry.io
+MARVIN_DATA_PATH?=$(HOME)/marvin/data
+MARVIN_ENGINE_NAME?=marvin_iris_automl
+MARVIN_TOOLBOX_VERSION?=0.0.5
+
+help:
+ @echo " marvin"
+ @echo " Prepare project to be used as a marvin package."
+ @echo " marvin-prod"
+ @echo " Prepare project to be used in production environment."
+ @echo " marvin-automl"
+ @echo " Prepare project to be used as a marvin-automl package."
+ @echo " update"
+ @echo " Reinstall requirements and setup.py dependencies."
+ @echo " clean"
+ @echo " Remove all generated artifacts."
+ @echo " clean-pyc"
+ @echo " Remove python artifacts."
+ @echo " clean-build"
+ @echo " Remove build artifacts."
+ @echo " clean-reports"
+ @echo " Remove coverage reports."
+ @echo " clean-deps"
+ @echo " Remove marvin setup.py dependencies."
+ @echo " docker-build"
+ @echo " Runs the docker build command with marvin env default parameters."
+ @echo " docker-push"
+ @echo " Runs the docker push command with marvin env default parameters."
+ @echo " docker-run"
+ @echo " Runs the docker run command with marvin env default parameters."
+
+marvin:
+ pip install -e ".[testing]"
+ marvin --help
+marvin-automl:
+ pip install -e ".[testing]"
+ bash install_automl.sh
+ marvin --help
+marvin-prod:
+ pip install .
+ marvin --help
+
+update:
+ pip install -e . -U
+
+clean-pyc:
+ find . -name '*.pyc' -exec rm -f {} +
+ find . -name '*.pyo' -exec rm -f {} +
+ find . -name '*~' -exec rm -f {} +
+
+clean-build:
+ rm -rf *.egg-info
+ rm -rf .cache
+ rm -rf .eggs
+ rm -rf dist
+ rm -rf build
+
+clean-reports:
+ rm -rf coverage_report/
+ rm -f coverage.xml
+ rm -f .coverage
+
+clean-deps:
+ pip freeze | grep -v "^-e" | xargs pip uninstall -y
+
+clean: clean-build clean-pyc clean-reports clean-deps
+
+docker-build: clean-build
+ mkdir -p build
+ tar -cf build/engine.tar --exclude=*.log --exclude=*.pkl --exclude='build' --exclude='notebooks' --exclude=*.tar *
+ cp -f $(MARVIN_DATA_PATH)/marvin-engine-executor-assembly-$(MARVIN_TOOLBOX_VERSION).jar build/marvin-engine-executor-assembly.jar
+ sudo docker build -t $(DOCKER_REGISTRY_ADRESS)/$(MARVIN_ENGINE_NAME):$(DOCKER_VERSION) .
+
+docker-run:
+ sudo docker run --name=marvin-$(MARVIN_ENGINE_NAME)-$(DOCKER_VERSION) --mount type=bind,source=$(MARVIN_DATA_PATH),destination=/marvin-data -p 8000:8000 $(DOCKER_REGISTRY_ADRESS)/$(MARVIN_ENGINE_NAME):$(DOCKER_VERSION)
+
+docker-push:
+ sudo docker push $(DOCKER_REGISTRY_ADRESS)/$(MARVIN_ENGINE_NAME):$(DOCKER_VERSION)
\ No newline at end of file
diff --git a/public-engines/iris-automl/README.md b/public-engines/iris-automl/README.md
new file mode 100644
index 0000000..5f7fc91
--- /dev/null
+++ b/public-engines/iris-automl/README.md
@@ -0,0 +1,151 @@
+# iris_automl_engine v0.0.1
+
+## Overview
+
+Iris Species classification engine from Kaggle with auto-sklearn (https://automl.github.io/auto-sklearn) and tpot (http://epistasislab.github.io/tpot/)
+
+
+## Requirements
+
+ - Python 3.7
+ - Numpy 1.11.0 or higher
+ - tpot
+ - auto-sklearn
+ - swig (Dependence of auto-sklearn)
+ - dialog (Dependence of install process)
+
+
+## Installation
+
+Create a new env with all dependences, replace PATH for iris-automl-engine path:
+
+ marvin engine-generateenv PATH
+
+Select tpot and auto-sklearn in dialog window after run engine-generateenv.
+
+Use the Marvin toolbox to provision, deploy and start the remote HTTP server.
+
+Edit the `marvin.ini` file, setting the options within the
+`ssh_deployment` section:
+
+1. `host`: the host IP address or name where the engine should be deployed. You
+can enable multi-host deployment using `,` to separate hosts
+2. `port`: the SSH connection port
+3. `user`: the SSH connection username. Currently, only a single user is
+supported. This user should be capable of *passwordless sudo*, although it can
+use password for the SSH connection
+
+Next, ensure that the remotes servers are provisioned (all required software
+are installed):
+
+ marvin engine-deploy --provision
+
+Next, package your engine:
+
+ marvin engine-deploy --package
+
+This will create a compressed archive containing your engine code under the
+`.packages` directory.
+
+Next, deploy your engine to remotes servers:
+
+ marvin engine-deploy
+
+By default, a dependency clean will be executed at each deploy. You can skip it
+using:
+
+ marvin engine-deploy --skip-clean
+
+Next, you can start the HTTP server in the remotes servers:
+
+ marvin engine-httpserver-remote start
+
+You can check if the HTTP server is running:
+
+ marvin engine-httpserver-remote status
+
+And stop it:
+
+ marvin engine-httpserver-remote stop
+
+After starting, you can test it by making a HTTP request to any endpoint, like:
+
+ curl -v http://example.com/predictor/health
+
+Under the hood, this engine uses Fabric to define provisioning and deployment
+process. Check the `fabfile.py` for more information. You can add new tasks or
+edit existing ones to match your provisioning and deployment pipeline.
+
+## Development
+
+### Getting started
+
+First, create a new virtualenv
+
+```
+mkvirtualenv marvin_iris_automl_engine_env
+```
+
+Now install the development dependencies
+
+```
+make marvin
+```
+
+You are now ready to code.
+
+
+### Adding new dependencies
+
+It\`s very important. All development dependencies should be added to `setup.py`.
+
+### Running tests
+
+This project uses *[py.test](http://pytest.org/)* as test runner and *[Tox](https://tox.readthedocs.io)* to manage virtualenvs.
+
+To run all tests use the following command
+
+```
+marvin test
+```
+
+To run specific test
+
+```
+marvin test tests/test_file.py::TestClass::test_method
+```
+
+
+### Writting documentation
+
+The project documentation is written using *[Jupyter](http://jupyter.readthedocs.io/)* notebooks.
+You can start the notebook server from the command line by running the following command
+
+```
+marvin notebook
+```
+
+Use notebooks to demonstrate how to use the lib features. It can also be useful to show some use cases.
+
+
+### Bumping version
+
+```
+marvin pkg-bumpversion [patch|minor|major]
+git add . && git commit -m "Bump version"
+```
+
+
+### Tagging version
+
+```
+marvin pkg-createtag
+git push origin master --follow-tags
+```
+
+
+### Logging
+
+The default log level is set to _WARNING_. You can change the log level at runtime setting another value to one of the following environment variable: `MARVIN_IRIS_AUTOML_ENGINE_LOG_LEVEL` or `LOG_LEVEL`. The available values are _CRITICAL_, _ERROR_, _WARNING_, _INFO_ and _DEBUG_.
+
+Be careful using `LOG_LEVEL`, it may affect another lib.
diff --git a/public-engines/iris-automl/docs.yaml b/public-engines/iris-automl/docs.yaml
new file mode 100644
index 0000000..d64c439
--- /dev/null
+++ b/public-engines/iris-automl/docs.yaml
@@ -0,0 +1,567 @@
+openapi: "3.0.0"
+info:
+ version: 0.0.1
+ title: marvin_iris_automl API Doc
+ contact:
+ name: mantainer Marvin AI Community
+ email: dev@marvin.apache.org
+ url: https://marvin.apache.org
+ license:
+ name: Apache License 2.0
+servers:
+ - url: http://localhost:8000
+ - url: http://0.0.0.0:8000
+tags:
+ - name: Docker
+ description: For Docker users, please use "make docker-build" and "make docker-run" commands in your engine virtualenv to start the server
+ - name: Acquisitor
+ description: Setup the initial_dataset with all cleaned data necessary to build your dataset in the next action
+ - name: Tpreparator
+ description: Setup the dataset with the transformed data that is compatible with the algorithm used to build the model in the next action
+ - name: Trainer
+ description: Setup the model with the result of the algorithm used to training
+ - name: Evaluator
+ description: Setup the metrics with the result of the algorithms used to test the model
+ - name: Predictor
+ description: Return the predicted value in a json parsable object format
+ - name: Feedback
+ description: Receive feedback message, user can manipulate this message for any use
+ - name: Pipeline
+ description: Perform all batch actions in your right order
+paths:
+ /acquisitor/health:
+ get:
+ summary: Get acquisitor's service health
+ operationId: getAcquisitorHealth
+ tags:
+ - Acquisitor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /acquisitor/status:
+ get:
+ summary: Get acquisitor's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getAcquisitorStatus
+ tags:
+ - Acquisitor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /acquisitor:
+ post:
+ summary: Run acquisitor
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: acquisitor
+ tags:
+ - Acquisitor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator/health:
+ get:
+ summary: Get trainer preparator's service health
+ operationId: getTPreparatortorHealth
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator/status:
+ get:
+ summary: Get trainer preparator's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getTPreparatorStatus
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator/reload?protocol=:
+ put:
+ summary: Reload artifact for trainer preparator
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadTPreparator
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator:
+ post:
+ summary: Run trainer preparator
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: tpreparator
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer/health:
+ get:
+ summary: Get trainer's service health
+ operationId: getTrainerHealth
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer/status:
+ get:
+ summary: Get trainer's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getTrainerStatus
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer/reload?protocol=:
+ put:
+ summary: Reload artifact for trainer
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadTrainer
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer:
+ post:
+ summary: Run trainer
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: trainer
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/health:
+ get:
+ summary: Get evaluator's service health
+ operationId: getEvaluatorHealth
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/status:
+ get:
+ summary: Get evaluator's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getEvaluatorStatus
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/metrics?protocol=:
+ get:
+ summary: Get metrics's value
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: Metrics protocol value
+ operationId: getMetrics
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/reload?protocol=:
+ put:
+ summary: Reload artifact for evaluator
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: reloadEvaluator
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator:
+ post:
+ summary: Run evaluator
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: evaluator
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor/health:
+ get:
+ summary: Get predictor's service health
+ operationId: getPredictorHealth
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor/status:
+ get:
+ summary: Get predictor's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getPredictorStatus
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor/reload?protocol=:
+ put:
+ summary: Reload artifact for predictor
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadPredictor
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor:
+ post:
+ summary: Run predictor
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: predictor
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback/health:
+ get:
+ summary: Get feedback's service health
+ operationId: getFeedbackHealth
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback/status:
+ get:
+ summary: Get feedback's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getFeedbackStatus
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback/reload?protocol=:
+ put:
+ summary: Reload artifact for feedback
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadFeedback
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback:
+ post:
+ summary: Run feedback
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: feedback
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /pipeline:
+ post:
+ summary: Do all batch actions (from Acquisitor to Evaluator)
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: pipeline
+ tags:
+ - Pipeline
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
\ No newline at end of file
diff --git a/public-engines/iris-automl/engine.messages b/public-engines/iris-automl/engine.messages
new file mode 100644
index 0000000..3d8c0f5
--- /dev/null
+++ b/public-engines/iris-automl/engine.messages
@@ -0,0 +1 @@
+[[1,2,3,4]]
diff --git a/public-engines/iris-automl/engine.metadata b/public-engines/iris-automl/engine.metadata
new file mode 100644
index 0000000..71ca401
--- /dev/null
+++ b/public-engines/iris-automl/engine.metadata
@@ -0,0 +1,62 @@
+{
+ "name": "marvin_iris_automl",
+ "version": "v0.0.1",
+ "engineType": "python",
+ "artifactsRemotePath": "/tmp/marvin",
+ "artifactManagerType": "FS",
+ "onlineActionTimeout": 1000,
+ "metricsTimeout": 10000,
+ "healthCheckTimeout": 2000,
+ "reloadTimeout": 600000,
+ "batchActionTimeout": 600000,
+ "pipelineActions": ["acquisitor", "tpreparator", "trainer", "evaluator"],
+ "actions": [{
+ "name": "acquisitor",
+ "actionType": "batch",
+ "port": 50051,
+ "host": "localhost",
+ "artifactsToPersist": ["initialdataset"],
+ "artifactsToLoad": [],
+ "pipeline": []
+ }, {
+ "name": "tpreparator",
+ "actionType": "batch",
+ "port": 50052,
+ "host": "localhost",
+ "artifactsToPersist": ["dataset"],
+ "artifactsToLoad": ["initialdataset"],
+ "pipeline": []
+ }, {
+ "name": "trainer",
+ "actionType": "batch",
+ "port": 50053,
+ "host": "localhost",
+ "artifactsToPersist": ["model"],
+ "artifactsToLoad": ["dataset"],
+ "pipeline": []
+ }, {
+ "name": "evaluator",
+ "actionType": "batch",
+ "port": 50054,
+ "host": "localhost",
+ "artifactsToPersist": ["metrics"],
+ "artifactsToLoad": ["dataset", "model"],
+ "pipeline": []
+ }, {
+ "name": "predictor",
+ "actionType": "online",
+ "port": 50055,
+ "host": "localhost",
+ "artifactsToPersist": [],
+ "artifactsToLoad": ["model", "metrics"],
+ "pipeline": ["ppreparator"]
+ }, {
+ "name": "feedback",
+ "actionType": "online",
+ "port": 50056,
+ "host": "localhost",
+ "artifactsToPersist": [],
+ "artifactsToLoad": [],
+ "pipeline": []
+ }]
+}
\ No newline at end of file
diff --git a/public-engines/iris-automl/engine.params b/public-engines/iris-automl/engine.params
new file mode 100644
index 0000000..410f586
--- /dev/null
+++ b/public-engines/iris-automl/engine.params
@@ -0,0 +1,3 @@
+{
+ "PARAM_1" : "VALUE_OF_PARAM_1"
+}
\ No newline at end of file
diff --git a/public-engines/iris-automl/fabfile.py b/public-engines/iris-automl/fabfile.py
new file mode 100644
index 0000000..851959f
--- /dev/null
+++ b/public-engines/iris-automl/fabfile.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+
+from fabric.api import env
+from fabric.api import run
+from fabric.api import execute
+from fabric.api import cd
+from fabric.api import local
+from fabric.api import put
+from fabric.api import sudo
+from fabric.state import output
+from marvin_python_toolbox import __version__ as TOOLBOX_VERSION
+from marvin_python_toolbox.common.config import Config
+
+_host = Config.get("host", section="ssh_deployment").split(",")
+_port = Config.get("port", section="ssh_deployment")
+_user = Config.get("user", section="ssh_deployment")
+
+for h in _host:
+ env.hosts.append("{user}@{host}:{port}".format(user=_user, host=h, port=_port))
+
+output["everything"] = False
+output["running"] = True
+
+env.package = "marvin_iris_automl"
+env.margin_engine_executor_prefix = "/opt/marvin/engine-executor"
+env.margin_engine_executor_jar = "marvin-engine-executor-assembly-{version}.jar".format(version=TOOLBOX_VERSION)
+env.marvin_engine_executor_path = env.margin_engine_executor_prefix + "/" + env.margin_engine_executor_jar
+
+
+def install_oracle_jdk():
+ sudo("add-apt-repository ppa:webupd8team/java -y")
+ sudo("apt-get -qq update")
+ run("echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections")
+ run("echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections")
+ sudo("apt-get install -y oracle-java8-installer")
+
+
+def install_virtualenvwrapper():
+ run("pip install virtualenvwrapper")
+ run("echo 'export WORKON_HOME=${HOME}/.virtualenvs' >> ${HOME}/.profile")
+ run("echo 'source /usr/local/bin/virtualenvwrapper.sh' >> ${HOME}/.profile")
+
+
+def install_apache_spark():
+ run("curl https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz -o /tmp/spark-2.1.1-bin-hadoop2.6.tgz")
+ sudo("tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/")
+ sudo("ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark")
+ run("echo 'export SPARK_HOME=/opt/spark' >> ${HOME}/.profile")
+
+
+def install_required_packages():
+ sudo("apt-get update -y")
+ sudo("apt-get install -y git")
+ sudo("apt-get install -y wget")
+ sudo("apt-get install -y python2.7-dev")
+ sudo("apt-get install -y python-pip")
+ sudo("apt-get install -y ipython")
+ sudo("apt-get install -y libffi-dev")
+ sudo("apt-get install -y libssl-dev")
+ sudo("apt-get install -y libxml2-dev")
+ sudo("apt-get install -y libxslt1-dev")
+ sudo("apt-get install -y libpng12-dev")
+ sudo("apt-get install -y libfreetype6-dev")
+ sudo("apt-get install -y python-tk")
+ sudo("apt-get install -y libsasl2-dev")
+ sudo("apt-get install -y python-pip")
+ sudo("apt-get install -y graphviz")
+ sudo("pip install --upgrade pip")
+
+
+def install_marvin_engine_executor():
+ sudo("mkdir -p {prefix}".format(prefix=env.margin_engine_executor_prefix))
+ with cd("{prefix}".format(prefix=env.margin_engine_executor_prefix)):
+ sudo("wget https://s3.amazonaws.com/marvin-engine-executor/{jar}".format(jar=env.margin_engine_executor_jar))
+
+
+def create_marvin_engines_prefix():
+ sudo("mkdir -p /opt/marvin/engines")
+ sudo("chown {user}:{user} /opt/marvin/engines".format(user=env.user))
+ sudo("mkdir -p /var/log/marvin/engines")
+ sudo("chown {user}:{user} /var/log/marvin/engines".format(user=env.user))
+ sudo("mkdir -p /var/run/marvin/engines")
+ sudo("chown {user}:{user} /var/run/marvin/engines".format(user=env.user))
+
+
+def configure_marvin_environment():
+ run("echo 'export MARVIN_HOME=${HOME}/marvin' >> ${HOME}/.profile")
+ run("echo 'export MARVIN_DATA_PATH=${MARVIN_HOME}/data' >> ${HOME}/.profile")
+ run("mkdir -p ${MARVIN_HOME}")
+ run("mkdir -p ${MARVIN_DATA_PATH}")
+
+
+def provision():
+ execute(install_required_packages)
+ execute(install_virtualenvwrapper)
+ execute(install_oracle_jdk)
+ execute(install_apache_spark)
+ execute(install_marvin_engine_executor)
+ execute(create_marvin_engines_prefix)
+ execute(configure_marvin_environment)
+
+
+def package(version):
+ package = env.package
+ local("mkdir -p .packages")
+ local("tar czvf .packages/{package}-{version}.tar.gz --exclude='.packages' .".format(
+ package=package, version=version))
+
+
+def deploy(version, skip_clean=False):
+ execute(engine_stop)
+ package = env.package
+ put(local_path=".packages/{package}-{version}.tar.gz".format(
+ package=package, version=version), remote_path="/tmp/")
+ run("mkdir -p /opt/marvin/engines/{package}/{version}".format(
+ package=package, version=version))
+ with cd("/opt/marvin/engines/{package}/{version}".format(
+ package=package, version=version)):
+ run("tar xzvf /tmp/{package}-{version}.tar.gz".format(
+ package=package, version=version))
+ with cd("/opt/marvin/engines/{package}".format(package=package)):
+ symlink_exists = run("stat current", quiet=True).succeeded
+ if (symlink_exists):
+ run("rm current")
+ run("ln -s {version} current".format(version=version))
+ with cd("/opt/marvin/engines/{package}/current".format(package=package)):
+ run("mkvirtualenv {package}_env".format(package=package))
+ run("setvirtualenvproject")
+ if skip_clean:
+ run("workon {package}_env && make marvin".format(
+ package=package))
+ else:
+ run("workon {package}_env && make clean && make marvin".format(
+ package=package))
+ execute(engine_start)
+
+
+def engine_start(http_host, http_port):
+ package = env.package
+
+ command = (
+ "workon {package}_env &&"
+ " (marvin engine-httpserver"
+ " -h {http_host}"
+ " -p {http_port}"
+ " -e {executor}"
+ " 1> /var/log/marvin/engines/{package}.out"
+ " 2> /var/log/marvin/engines/{package}.err"
+ " & echo $! > /var/run/marvin/engines/{package}.pid)"
+ ).format(
+ package=package,
+ http_host=http_host,
+ http_port=http_port,
+ executor=env.marvin_engine_executor_path
+ )
+
+ with cd("/opt/marvin/engines/{package}/current".format(package=package)):
+ run(command, pty=False)
+
+
+def engine_stop():
+ package = env.package
+
+ pid_file_exists = run("cat /var/run/marvin/engines/{package}.pid".format(
+ package=package), quiet=True)
+ if pid_file_exists.succeeded:
+ with cd("/opt/marvin/engines/{package}/current".format(package=package)):
+ children_pids = run("ps --ppid $(cat /var/run/marvin/engines/{package}.pid) -o pid --no-headers |xargs echo".format(
+ package=package))
+ run("kill $(cat /var/run/marvin/engines/{package}.pid) {children_pids}".format(
+ package=package, children_pids=children_pids))
+ run("rm /var/run/marvin/engines/{package}.pid".format(package=package))
+
+
+def engine_status():
+ package = env.package
+ pid_file_exists = run("cat /var/run/marvin/engines/{package}.pid".format(
+ package=package), quiet=True)
+ if pid_file_exists.succeeded:
+ is_running = run("ps $(cat /var/run/marvin/engines/{package}.pid)".format(package=package), quiet=True)
+ if is_running.succeeded:
+ print "Your engine is running :)"
+ else:
+ print "Your engine is not running :("
+ else:
+ print "Your engine is not running :("
\ No newline at end of file
diff --git a/public-engines/iris-automl/feedback.messages b/public-engines/iris-automl/feedback.messages
new file mode 100644
index 0000000..5ff7ec4
--- /dev/null
+++ b/public-engines/iris-automl/feedback.messages
@@ -0,0 +1,3 @@
+[{
+ "msg1": "Hello from marvin engine!"
+}]
\ No newline at end of file
diff --git a/public-engines/iris-automl/install_automl.sh b/public-engines/iris-automl/install_automl.sh
new file mode 100644
index 0000000..bc73021
--- /dev/null
+++ b/public-engines/iris-automl/install_automl.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+###############################################
+# Marvin AutoML installation Script #
+###############################################
+cmd=(dialog --title "Marvin AutoML" --separate-output --checklist "Select tools:" 22 76 16)
+options=(1 "auto-sklearn" off # any option can be set to default to "on"
+ 2 "h2o AutoML" off
+ 3 "TPOT" off)
+choices=$("${cmd[@]}" "${options[@]}" 2>&1 >/dev/tty)
+clear
+for choice in $choices
+do
+ case $choice in
+ 1)
+ echo "Instaling auto-sklearn..."
+ wget https://raw.githubusercontent.com/automl/auto-sklearn/master/requirements.txt \
+ | xargs -n 1 -L 1 pip install
+ pip install auto-sklearn
+ ;;
+ 2)
+ echo "Installing h2o..."
+ pip install requests
+ pip install tabulate
+ pip install scikit-learn
+ pip install http://h2o-release.s3.amazonaws.com/h2o/rel-yau/3/Python/h2o-3.26.0.3-py2.py3-none-any.whl
+ wget http://h2o-release.s3.amazonaws.com/h2o/rel-yau/3/h2o-3.26.0.3.zip
+ unzip h2o-3.26.0.3.zip
+ rm h2o-3.26.0.3.zip
+ ;;
+ 3)
+ echo "Installing TPOT..."
+ pip install tpot
+ ;;
+ esac
+done
\ No newline at end of file
diff --git a/public-engines/iris-automl/marvin.ini b/public-engines/iris-automl/marvin.ini
new file mode 100644
index 0000000..723eaee
--- /dev/null
+++ b/public-engines/iris-automl/marvin.ini
@@ -0,0 +1,11 @@
+[marvin]
+package = marvin_iris_automl
+type = automl-engine
+executor_url = https://s3.amazonaws.com/marvin-engine-executor/marvin-engine-executor-assembly-0.0.5.jar
+
+[ssh_deployment]
+# You can enable multi-host deployment like this
+# host = host1.com,host2.com,hostN.com
+host = host1.com
+port = 22
+user = marvin
\ No newline at end of file
diff --git a/public-engines/iris-automl/marvin_iris_automl/VERSION b/public-engines/iris-automl/marvin_iris_automl/VERSION
new file mode 100644
index 0000000..8a9ecc2
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/VERSION
@@ -0,0 +1 @@
+0.0.1
\ No newline at end of file
diff --git a/public-engines/iris-automl/marvin_iris_automl/__init__.py b/public-engines/iris-automl/marvin_iris_automl/__init__.py
new file mode 100644
index 0000000..e715c57
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/__init__.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os.path
+
+from .data_handler import *
+from .prediction import *
+from .training import *
+
+
+# Get package version number from "VERSION" file
+with open(os.path.join(os.path.dirname(__file__), 'VERSION'), 'rb') as f:
+ __version__ = f.read().decode('ascii').strip()
diff --git a/public-engines/iris-automl/marvin_iris_automl/_compatibility.py b/public-engines/iris-automl/marvin_iris_automl/_compatibility.py
new file mode 100644
index 0000000..b2e635d
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/_compatibility.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Compatibility module.
+
+Import this module to help to write code compatible with Python 2 and 3.
+"""
+
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import six
+
+__all__ = ['six']
+
+# Add here any code that have to differentiate between python 2 and 3.
diff --git a/public-engines/iris-automl/marvin_iris_automl/_logging.py b/public-engines/iris-automl/marvin_iris_automl/_logging.py
new file mode 100644
index 0000000..9e464a3
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/_logging.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Custom logging module.
+
+This module is responsible to manage log messages and log file.
+"""
+
+import sys
+import os
+import os.path
+import logging
+
+DEFAULT_LOG_LEVEL = logging.WARNING
+DEFAULT_LOG_DIR = '/tmp'
+
+
+class Logger(logging.getLoggerClass()):
+ """Custom logger class.
+
+ Use this class to customize the logger behavior or to intercept the
+ messages.
+ """
+ def error(self, msg, *args, **kwargs):
+ # Add here code to intercept the project error messages
+ super(Logger, self).error(msg, *args, **kwargs)
+
+ def critical(self, msg, *args, **kwargs):
+ # Add here code to intercept the project critical messages
+ super(Logger, self).critical(msg, *args, **kwargs)
+
+
+logging.setLoggerClass(Logger)
+
+
+def get_logger(name, namespace='marvin_iris_automl',
+ log_level=DEFAULT_LOG_LEVEL, log_dir=DEFAULT_LOG_DIR):
+ """Build a logger that outputs to a file and to the console,"""
+
+ log_level = (os.getenv('{}_LOG_LEVEL'.format(namespace.upper())) or
+ os.getenv('LOG_LEVEL', log_level))
+ log_dir = (os.getenv('{}_LOG_DIR'.format(namespace.upper())) or
+ os.getenv('LOG_DIR', log_dir))
+
+ logger = logging.getLogger('{}.{}'.format(namespace, name))
+ logger.setLevel(log_level)
+
+ formatter = logging.Formatter(
+ '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+ # Create a console stream handler
+ console_handler = logging.StreamHandler()
+ console_handler.setLevel(log_level)
+ console_handler.setFormatter(formatter)
+ logger.addHandler(console_handler)
+
+ try:
+ if log_dir:
+ log_path = os.path.abspath(log_dir)
+ log_filename = '{name}.{pid}.log'.format(
+ name=namespace, pid=os.getpid())
+
+ file_path = str(os.path.join(log_path, log_filename))
+
+ if not os.path.exists(log_path):
+ os.makedirs(log_path, mode=774)
+
+ # Create a file handler
+ file_handler = logging.FileHandler(file_path)
+ file_handler.setLevel(log_level)
+ file_handler.setFormatter(formatter)
+ logger.addHandler(file_handler)
+ except OSError as e:
+ logger.error('Could not create log file {file}: {error}'.format(
+ file=file_path, error=e.strerror))
+
+ return logger
diff --git a/public-engines/iris-automl/marvin_iris_automl/data_handler/__init__.py b/public-engines/iris-automl/marvin_iris_automl/data_handler/__init__.py
new file mode 100644
index 0000000..677dd17
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/data_handler/__init__.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+from .acquisitor_and_cleaner import AcquisitorAndCleaner
+from .training_preparator import TrainingPreparator
\ No newline at end of file
diff --git a/public-engines/iris-automl/marvin_iris_automl/data_handler/acquisitor_and_cleaner.py b/public-engines/iris-automl/marvin_iris_automl/data_handler/acquisitor_and_cleaner.py
new file mode 100644
index 0000000..1a8e51d
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/data_handler/acquisitor_and_cleaner.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""AcquisitorAndCleaner engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBaseDataHandler
+
+__all__ = ['AcquisitorAndCleaner']
+
+
+logger = get_logger('acquisitor_and_cleaner')
+
+
+class AcquisitorAndCleaner(EngineBaseDataHandler):
+
+ def __init__(self, **kwargs):
+ super(AcquisitorAndCleaner, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ from sklearn import datasets
+ iris = datasets.load_iris()
+ self.marvin_initial_dataset = iris
+
diff --git a/public-engines/iris-automl/marvin_iris_automl/data_handler/training_preparator.py b/public-engines/iris-automl/marvin_iris_automl/data_handler/training_preparator.py
new file mode 100644
index 0000000..cc63e91
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/data_handler/training_preparator.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""TrainingPreparator engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBaseDataHandler
+
+__all__ = ['TrainingPreparator']
+
+
+logger = get_logger('training_preparator')
+
+
+class TrainingPreparator(EngineBaseDataHandler):
+
+ def __init__(self, **kwargs):
+ super(TrainingPreparator, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ import sklearn.model_selection
+ X, y = self.marvin_initial_dataset.data, self.marvin_initial_dataset.target
+ X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, random_state=1)
+
+ self.marvin_dataset = {'train_X': X_train, 'train_y': y_train, 'test_X': X_test, 'test_y': y_test}
+
diff --git a/public-engines/iris-automl/marvin_iris_automl/prediction/__init__.py b/public-engines/iris-automl/marvin_iris_automl/prediction/__init__.py
new file mode 100644
index 0000000..e9b3c7c
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/prediction/__init__.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+from .prediction_preparator import PredictionPreparator
+from .predictor import Predictor
+from .feedback import Feedback
\ No newline at end of file
diff --git a/public-engines/iris-automl/marvin_iris_automl/prediction/feedback.py b/public-engines/iris-automl/marvin_iris_automl/prediction/feedback.py
new file mode 100644
index 0000000..085808b
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/prediction/feedback.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2019] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Feedback engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBasePrediction
+
+__all__ = ['Feedback']
+
+
+logger = get_logger('feedback')
+
+
+class Feedback(EngineBasePrediction):
+
+ def __init__(self, **kwargs):
+ super(Feedback, self).__init__(**kwargs)
+
+ def execute(self, input_message, params, **kwargs):
+ """
+ Receive feedback message, user can manipulate this message for any use.
+ Return "Done" to signal that the message is received and processed.
+ """
+ return {"message": "Done"}
\ No newline at end of file
diff --git a/public-engines/iris-automl/marvin_iris_automl/prediction/prediction_preparator.py b/public-engines/iris-automl/marvin_iris_automl/prediction/prediction_preparator.py
new file mode 100644
index 0000000..9745020
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/prediction/prediction_preparator.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""PredictionPreparator engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBasePrediction
+
+__all__ = ['PredictionPreparator']
+
+
+logger = get_logger('prediction_preparator')
+
+
+class PredictionPreparator(EngineBasePrediction):
+
+ def __init__(self, **kwargs):
+ super(PredictionPreparator, self).__init__(**kwargs)
+
+ def execute(self, input_message, params, **kwargs):
+ import numpy as np
+ input_message = np.array([input_message])
+
+ return input_message
diff --git a/public-engines/iris-automl/marvin_iris_automl/prediction/predictor.py b/public-engines/iris-automl/marvin_iris_automl/prediction/predictor.py
new file mode 100644
index 0000000..424eaf9
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/prediction/predictor.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Predictor engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBasePrediction
+
+__all__ = ['Predictor']
+
+
+logger = get_logger('predictor')
+
+
+class Predictor(EngineBasePrediction):
+
+ def __init__(self, **kwargs):
+ super(Predictor, self).__init__(**kwargs)
+
+ def execute(self, input_message, params, **kwargs):
+ from sklearn import datasets
+ iris = datasets.load_iris()
+ prediction = self.marvin_model.predict(input_message)
+ final_prediction = iris.target_names[prediction][0]
+
+ return final_prediction
diff --git a/public-engines/iris-automl/marvin_iris_automl/training/__init__.py b/public-engines/iris-automl/marvin_iris_automl/training/__init__.py
new file mode 100644
index 0000000..e1723b7
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/training/__init__.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+from .metrics_evaluator import MetricsEvaluator
+from .trainer import Trainer
\ No newline at end of file
diff --git a/public-engines/iris-automl/marvin_iris_automl/training/metrics_evaluator.py b/public-engines/iris-automl/marvin_iris_automl/training/metrics_evaluator.py
new file mode 100644
index 0000000..22f891a
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/training/metrics_evaluator.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""MetricsEvaluator engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBaseTraining
+
+__all__ = ['MetricsEvaluator']
+
+
+logger = get_logger('metrics_evaluator')
+
+
+class MetricsEvaluator(EngineBaseTraining):
+
+ def __init__(self, **kwargs):
+ super(MetricsEvaluator, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ from sklearn import metrics
+ predictions = self.marvin_model.predict(self.marvin_dataset['test_X'])
+ self.marvin_metrics = metrics.accuracy_score(self.marvin_dataset['test_y'], predictions)
+
diff --git a/public-engines/iris-automl/marvin_iris_automl/training/trainer.py b/public-engines/iris-automl/marvin_iris_automl/training/trainer.py
new file mode 100644
index 0000000..7eee0e9
--- /dev/null
+++ b/public-engines/iris-automl/marvin_iris_automl/training/trainer.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Trainer engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_toolbox.engine_base import EngineBaseTraining
+
+__all__ = ['Trainer']
+
+
+logger = get_logger('trainer')
+
+
+class Trainer(EngineBaseTraining):
+
+ def __init__(self, **kwargs):
+ super(Trainer, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ import autosklearn.classification
+
+ automl = autosklearn.classification.AutoSklearnClassifier(
+ time_left_for_this_task=120,
+ per_run_time_limit=30,
+ resampling_strategy='cv',
+ resampling_strategy_arguments={'folds': 5},
+ )
+
+ automl.fit(self.marvin_dataset['train_X'].copy(), self.marvin_dataset['train_y'].copy())
+ automl.refit(self.marvin_dataset['train_X'].copy(), self.marvin_dataset['train_y'].copy())
+
+ # Using fitted_pipeline_ method to get model
+ self.marvin_model = automl
+
diff --git a/public-engines/iris-automl/notebooks/.ipynb_checkpoints/autosk_sample-checkpoint.ipynb b/public-engines/iris-automl/notebooks/.ipynb_checkpoints/autosk_sample-checkpoint.ipynb
new file mode 100644
index 0000000..cde2e0d
--- /dev/null
+++ b/public-engines/iris-automl/notebooks/.ipynb_checkpoints/autosk_sample-checkpoint.ipynb
@@ -0,0 +1,160 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Documentation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Sample"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "marvin_cell": "acquisitor"
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/bruno/.virtualenvs/iris-automl-engine-env/lib/python3.7/site-packages/sklearn/feature_extraction/text.py:17: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working\n",
+ " from collections import Mapping, defaultdict\n"
+ ]
+ }
+ ],
+ "source": [
+ "from sklearn import datasets\n",
+ "iris = datasets.load_iris()\n",
+ "marvin_initial_dataset = iris"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "marvin_cell": "tpreparator"
+ },
+ "outputs": [],
+ "source": [
+ "import sklearn.model_selection\n",
+ "X,y = marvin_initial_dataset.data,marvin_initial_dataset.target\n",
+ "X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, random_state=1)\n",
+ "\n",
+ "marvin_dataset = {'train_X': X_train, 'train_y': y_train, 'test_X': X_test, 'test_y': y_test}\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Cross validation example\n",
+ "\n",
+ "https://automl.github.io/auto-sklearn/master/examples/example_crossvalidation.html#sphx-glr-examples-example-crossvalidation-py"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "marvin_cell": "trainer"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[WARNING] [2019-09-11 13:10:45,749:EnsembleBuilder(1):f00fc4935fefc8f83f31eff3100a845d] No models better than random - using Dummy Score!\n",
+ "[WARNING] [2019-09-11 13:10:45,754:EnsembleBuilder(1):f00fc4935fefc8f83f31eff3100a845d] No models better than random - using Dummy Score!\n",
+ "[WARNING] [2019-09-11 13:10:47,758:EnsembleBuilder(1):f00fc4935fefc8f83f31eff3100a845d] No models better than random - using Dummy Score!\n",
+ "1\n",
+ "['/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000000.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000001.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000002.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000003.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000004.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000005.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000006.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000007.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000008.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000009.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000010.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000011.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000012.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000013.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000014.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000015.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000016.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000017.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000018.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000019.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000020.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000021.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000022.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000023.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000024.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000025.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000026.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000027.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000028.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000029.ensemble']\n"
+ ]
+ }
+ ],
+ "source": [
+ "import autosklearn.classification\n",
+ "\n",
+ "automl = autosklearn.classification.AutoSklearnClassifier(\n",
+ " time_left_for_this_task=120,\n",
+ " per_run_time_limit=30,\n",
+ " resampling_strategy='cv',\n",
+ " resampling_strategy_arguments={'folds': 5},\n",
+ ")\n",
+ "\n",
+ "automl.fit(marvin_dataset['train_X'].copy(), marvin_dataset['train_y'].copy())\n",
+ "automl.refit(marvin_dataset['train_X'].copy(), marvin_dataset['train_y'].copy())\n",
+ "\n",
+ "# Using fitted_pipeline_ method to get model\n",
+ "marvin_model = automl\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "marvin_cell": "evaluator"
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn import metrics\n",
+ "predictions = marvin_model.predict(marvin_dataset['test_X'])\n",
+ "marvin_metrics = metrics.accuracy_score(marvin_dataset['test_y'], predictions)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "marvin_cell": "ppreparator"
+ },
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "input_message = np.array([input_message])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "marvin_cell": "predictor"
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn import datasets\n",
+ "iris = datasets.load_iris()\n",
+ "prediction = marvin_model.predict(input_message)\n",
+ "final_prediction = iris.target_names[prediction][0]"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/public-engines/iris-automl/notebooks/autosk_sample.ipynb b/public-engines/iris-automl/notebooks/autosk_sample.ipynb
new file mode 100644
index 0000000..cde2e0d
--- /dev/null
+++ b/public-engines/iris-automl/notebooks/autosk_sample.ipynb
@@ -0,0 +1,160 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Documentation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Sample"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "marvin_cell": "acquisitor"
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/bruno/.virtualenvs/iris-automl-engine-env/lib/python3.7/site-packages/sklearn/feature_extraction/text.py:17: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working\n",
+ " from collections import Mapping, defaultdict\n"
+ ]
+ }
+ ],
+ "source": [
+ "from sklearn import datasets\n",
+ "iris = datasets.load_iris()\n",
+ "marvin_initial_dataset = iris"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "marvin_cell": "tpreparator"
+ },
+ "outputs": [],
+ "source": [
+ "import sklearn.model_selection\n",
+ "X,y = marvin_initial_dataset.data,marvin_initial_dataset.target\n",
+ "X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, random_state=1)\n",
+ "\n",
+ "marvin_dataset = {'train_X': X_train, 'train_y': y_train, 'test_X': X_test, 'test_y': y_test}\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Cross validation example\n",
+ "\n",
+ "https://automl.github.io/auto-sklearn/master/examples/example_crossvalidation.html#sphx-glr-examples-example-crossvalidation-py"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "marvin_cell": "trainer"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[WARNING] [2019-09-11 13:10:45,749:EnsembleBuilder(1):f00fc4935fefc8f83f31eff3100a845d] No models better than random - using Dummy Score!\n",
+ "[WARNING] [2019-09-11 13:10:45,754:EnsembleBuilder(1):f00fc4935fefc8f83f31eff3100a845d] No models better than random - using Dummy Score!\n",
+ "[WARNING] [2019-09-11 13:10:47,758:EnsembleBuilder(1):f00fc4935fefc8f83f31eff3100a845d] No models better than random - using Dummy Score!\n",
+ "1\n",
+ "['/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000000.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000001.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000002.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000003.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000004.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000005.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000006.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000007.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000008.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000009.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000010.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000011.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000012.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000013.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000014.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000015.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000016.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000017.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000018.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000019.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000020.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000021.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000022.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000023.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000024.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000025.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000026.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000027.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000028.ensemble', '/tmp/autosklearn_cv_example_tmp/.auto-sklearn/ensembles/1.0000000029.ensemble']\n"
+ ]
+ }
+ ],
+ "source": [
+ "import autosklearn.classification\n",
+ "\n",
+ "automl = autosklearn.classification.AutoSklearnClassifier(\n",
+ " time_left_for_this_task=120,\n",
+ " per_run_time_limit=30,\n",
+ " resampling_strategy='cv',\n",
+ " resampling_strategy_arguments={'folds': 5},\n",
+ ")\n",
+ "\n",
+ "automl.fit(marvin_dataset['train_X'].copy(), marvin_dataset['train_y'].copy())\n",
+ "automl.refit(marvin_dataset['train_X'].copy(), marvin_dataset['train_y'].copy())\n",
+ "\n",
+ "# Using fitted_pipeline_ method to get model\n",
+ "marvin_model = automl\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "marvin_cell": "evaluator"
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn import metrics\n",
+ "predictions = marvin_model.predict(marvin_dataset['test_X'])\n",
+ "marvin_metrics = metrics.accuracy_score(marvin_dataset['test_y'], predictions)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "marvin_cell": "ppreparator"
+ },
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "input_message = np.array([input_message])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "marvin_cell": "predictor"
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn import datasets\n",
+ "iris = datasets.load_iris()\n",
+ "prediction = marvin_model.predict(input_message)\n",
+ "final_prediction = iris.target_names[prediction][0]"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/public-engines/iris-automl/notebooks/tpot_sample.ipynb b/public-engines/iris-automl/notebooks/tpot_sample.ipynb
new file mode 100644
index 0000000..2cfba15
--- /dev/null
+++ b/public-engines/iris-automl/notebooks/tpot_sample.ipynb
@@ -0,0 +1,179 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Documentation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Sample"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sklearn import datasets\n",
+ "iris = datasets.load_iris()\n",
+ "marvin_initial_dataset = iris"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import sklearn.model_selection\n",
+ "X,y = marvin_initial_dataset.data,marvin_initial_dataset.target\n",
+ "X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, random_state=1)\n",
+ "\n",
+ "marvin_dataset = {'train_X': X_train, 'train_y': y_train, 'test_X': X_test, 'test_y': y_test}\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Warning: xgboost.XGBClassifier is not available and will not be used by TPOT.\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(IntProgress(value=0, description='Optimization Progress', max=120, style=ProgressStyle(descript…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Generation 1 - Current best internal CV score: 0.982213438735178\n",
+ "Generation 2 - Current best internal CV score: 0.982213438735178\n",
+ "Generation 3 - Current best internal CV score: 0.982213438735178\n",
+ "Generation 4 - Current best internal CV score: 0.982213438735178\n",
+ "Generation 5 - Current best internal CV score: 0.982213438735178\n",
+ "\n",
+ "Best pipeline: GaussianNB(Normalizer(input_matrix, norm=l2))\n"
+ ]
+ }
+ ],
+ "source": [
+ "from tpot import TPOTClassifier\n",
+ "\n",
+ "pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n",
+ " random_state=42, verbosity=2)\n",
+ "\n",
+ "pipeline_optimizer.fit(marvin_dataset['train_X'].copy(), marvin_dataset['train_y'].copy())\n",
+ "\n",
+ "# Using fitted_pipeline_ method to get model\n",
+ "marvin_model = pipeline_optimizer.fitted_pipeline_\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sklearn import metrics\n",
+ "predictions = marvin_model.predict(marvin_dataset['test_X'])\n",
+ "marvin_metrics = metrics.accuracy_score(marvin_dataset['test_y'], predictions)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "NameError",
+ "evalue": "name 'input_message' is not defined",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
+ "\u001b[0;32m<ipython-input-9-e2c2e30e02dc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0minput_message\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minput_message\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+ "\u001b[0;31mNameError\u001b[0m: name 'input_message' is not defined"
+ ]
+ }
+ ],
+ "source": [
+ "import numpy as np\n",
+ "input_message = np.array([input_message])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "NameError",
+ "evalue": "name 'input_message' is not defined",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
+ "\u001b[0;32m<ipython-input-10-7c6c94f5195d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdatasets\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0miris\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatasets\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_iris\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mprediction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmarvin_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_message\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mfinal_prediction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0miris\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtarget_names\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mprediction\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;31mNameError\u001b[0m: name 'input_message' is not defined"
+ ]
+ }
+ ],
+ "source": [
+ "from sklearn import datasets\n",
+ "iris = datasets.load_iris()\n",
+ "prediction = marvin_model.predict(input_message)\n",
+ "final_prediction = iris.target_names[prediction][0]\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/public-engines/iris-automl/pytest.ini b/public-engines/iris-automl/pytest.ini
new file mode 100644
index 0000000..cab8644
--- /dev/null
+++ b/public-engines/iris-automl/pytest.ini
@@ -0,0 +1,4 @@
+[pytest]
+minversion = 2.0
+norecursedirs = .git .tox .eggs .cache *.egg build dist tmp*
+python_files = test*.py
\ No newline at end of file
diff --git a/public-engines/iris-automl/requirements.txt b/public-engines/iris-automl/requirements.txt
new file mode 100644
index 0000000..983d39a
--- /dev/null
+++ b/public-engines/iris-automl/requirements.txt
@@ -0,0 +1,20 @@
+setuptools
+nose
+Cython
+
+numpy>=1.9.0
+scipy>=0.14.1
+
+scikit-learn>=0.21.0,<0.22
+
+lockfile
+joblib
+psutil
+pyyaml
+liac-arff
+pandas
+
+ConfigSpace>=0.4.0,<0.5
+pynisher>=0.4.2
+pyrfr>=0.7,<0.9
+smac==0.8
diff --git a/public-engines/iris-automl/setup.cfg b/public-engines/iris-automl/setup.cfg
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/public-engines/iris-automl/setup.cfg
diff --git a/public-engines/iris-automl/setup.py b/public-engines/iris-automl/setup.py
new file mode 100644
index 0000000..afe1d13
--- /dev/null
+++ b/public-engines/iris-automl/setup.py
@@ -0,0 +1,175 @@
+from __future__ import print_function
+
+import os
+import shutil
+from os.path import dirname, join
+from setuptools import setup, find_packages
+from setuptools.command.test import test as TestCommand
+from setuptools.command.develop import develop as _develop
+from setuptools.command.install import install as _install
+
+
+REQUIREMENTS_TESTS = [
+ 'pytest>=2.6.4',
+ 'pytest-cov>=1.8.1',
+ 'mock>=2.0.0',
+ 'virtualenv>=15.0.1',
+ 'tox>=2.2.0',
+]
+
+def _get_version():
+ """Return the project version from VERSION file."""
+
+ with open(join(dirname(__file__), 'marvin_iris_automl/VERSION'), 'rb') as f:
+ version = f.read().decode('ascii').strip()
+ return version
+
+
+def _hooks(dir):
+ _set_autocomplete()
+ _install_notebook_extension()
+
+
+def _set_autocomplete():
+ import marvin_python_toolbox as toolbox
+ virtualenv = os.environ.get('VIRTUAL_ENV', None)
+
+ if virtualenv:
+ postactivate = os.path.join(virtualenv, 'bin', 'postactivate')
+
+ if os.path.exists(postactivate):
+ shutil.copy(
+ os.path.join(toolbox.__path__[0], 'extras', 'marvin_bash_completion'),
+ os.path.join(virtualenv, 'marvin_bash_completion')
+ )
+
+ command = 'source "{}"'.format(os.path.join(virtualenv, 'marvin_bash_completion'))
+
+ with open(postactivate, 'r+') as fp:
+ lines = fp.readlines()
+ fp.seek(0)
+ configured = False
+ for line in lines:
+ if 'marvin_bash_completion' in line:
+ # Replacing old autocomplete configuration
+ fp.write(command)
+ configured = True
+ else:
+ fp.write(line)
+
+ if not configured:
+ fp.write(command)
+ # 'Autocomplete was successfully configured'
+ fp.write('\n')
+ fp.truncate()
+
+
+def _install_notebook_extension():
+ import marvin_python_toolbox as toolbox
+
+ install_command = [
+ "jupyter",
+ "nbextension",
+ "install",
+ os.path.join(toolbox.__path__[0], 'extras', 'notebook_extensions', 'main.js'),
+ "--destination",
+ "marvin.js",
+ "--sys-prefix",
+ "--overwrite"
+ ]
+
+ os.system(' '.join(install_command))
+
+ enable_command = [
+ "jupyter",
+ "nbextension",
+ "enable",
+ "marvin",
+ "--sys-prefix"
+ ]
+
+ os.system(' '.join(enable_command))
+
+
+class develop(_develop):
+ def run(self):
+ _develop.run(self)
+ self.execute(_hooks, (self.install_lib,), msg="Running develop preparation task")
+
+
+class install(_install):
+ def run(self):
+ _install.run(self)
+ self.execute(_hooks, (self.install_lib,), msg="Running install preparation task")
+
+
+class Tox(TestCommand):
+ """Run the test cases using TOX command."""
+
+ user_options = [('tox-args=', 'a', "Arguments to pass to tox")]
+
+ def initialize_options(self):
+ TestCommand.initialize_options(self)
+ self.tox_args = None
+
+ def finalize_options(self):
+ TestCommand.finalize_options(self)
+ self.test_args = []
+ self.test_suite = True
+
+ def run_tests(self):
+ # Import here, cause outside the eggs aren't loaded
+ import tox
+ import shlex
+ import sys
+ args = self.tox_args
+ if args:
+ args = shlex.split(self.tox_args)
+ else:
+ # Run all tests by default
+ args = ['-c', join(dirname(__file__), 'tox.ini'), 'tests']
+ errno = tox.cmdline(args=args)
+ sys.exit(errno)
+
+
+setup(
+ name='marvin_iris_automl',
+ version=_get_version(),
+ url='',
+ description='Marvin engine',
+ long_description=open(join(dirname(__file__), 'README.md')).read(),
+ author='Marvin AI Community',
+ maintainer='Marvin AI Community',
+ maintainer_email='dev@marvin.apache.org',
+ packages=find_packages(exclude=('tests', 'tests.*')),
+ include_package_data=True,
+ zip_safe=False,
+ classifiers=[
+ 'Development Status :: 3 - Alpha',
+ 'Intended Audience :: Developers',
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.3',
+ 'Programming Language :: Python :: 3.4',
+ 'Programming Language :: Python :: 3.5',
+ 'Topic :: Software Development :: Libraries :: Python Modules',
+ ],
+ install_requires=[
+ 'scikit-learn>=0.18.2',
+ 'scipy>=0.19.1',
+ 'numpy>=1.13.1',
+ 'pandas>=0.20.3',
+ 'matplotlib>=2.0.2',
+ 'marvin-python-toolbox>=0.0.5',
+ 'Fabric>=1.14.0',
+ ],
+ tests_require=REQUIREMENTS_TESTS,
+ extras_require={
+ 'testing': REQUIREMENTS_TESTS,
+ },
+ cmdclass={
+ 'test': Tox, 'develop': develop, 'install': install
+ },
+)
\ No newline at end of file
diff --git a/public-engines/iris-automl/tests/conftest.py b/public-engines/iris-automl/tests/conftest.py
new file mode 100644
index 0000000..efd37f3
--- /dev/null
+++ b/public-engines/iris-automl/tests/conftest.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os
+import pytest
+
+os.environ['TESTING'] = 'True'
+
+
+@pytest.fixture
+def mocked_params():
+ return {'params': 1}
diff --git a/public-engines/iris-automl/tests/data_handler/test_acquisitor_and_cleaner.py b/public-engines/iris-automl/tests/data_handler/test_acquisitor_and_cleaner.py
new file mode 100644
index 0000000..fcb794d
--- /dev/null
+++ b/public-engines/iris-automl/tests/data_handler/test_acquisitor_and_cleaner.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_automl.data_handler import AcquisitorAndCleaner
+
+
+class TestAcquisitorAndCleaner:
+ def test_execute(self, mocked_params):
+ ac = AcquisitorAndCleaner()
+ ac.execute(params=mocked_params)
+ assert not ac._params
diff --git a/public-engines/iris-automl/tests/data_handler/test_training_preparator.py b/public-engines/iris-automl/tests/data_handler/test_training_preparator.py
new file mode 100644
index 0000000..d151766
--- /dev/null
+++ b/public-engines/iris-automl/tests/data_handler/test_training_preparator.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_automl.data_handler import TrainingPreparator
+
+
+class TestTrainingPreparator:
+ def test_execute(self, mocked_params):
+ ac = TrainingPreparator()
+ ac.execute(params=mocked_params)
+ assert not ac._params
\ No newline at end of file
diff --git a/public-engines/iris-automl/tests/prediction/test_feedback.py b/public-engines/iris-automl/tests/prediction/test_feedback.py
new file mode 100644
index 0000000..3354b7b
--- /dev/null
+++ b/public-engines/iris-automl/tests/prediction/test_feedback.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_automl.prediction import Feedback
+
+
+class TestFeedback:
+ def test_execute(self, mocked_params):
+ fb = Feedback()
+ fb.execute(input_message="fake message", params=mocked_params)
+ assert not fb._params
\ No newline at end of file
diff --git a/public-engines/iris-automl/tests/prediction/test_prediction_preparator.py b/public-engines/iris-automl/tests/prediction/test_prediction_preparator.py
new file mode 100644
index 0000000..4006cd7
--- /dev/null
+++ b/public-engines/iris-automl/tests/prediction/test_prediction_preparator.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_automl.prediction import PredictionPreparator
+
+
+class TestPredictionPreparator:
+ def test_execute(self, mocked_params):
+ ac = PredictionPreparator()
+ ac.execute(input_message="fake message", params=mocked_params)
+ assert not ac._params
\ No newline at end of file
diff --git a/public-engines/iris-automl/tests/prediction/test_predictor.py b/public-engines/iris-automl/tests/prediction/test_predictor.py
new file mode 100644
index 0000000..46fddf7
--- /dev/null
+++ b/public-engines/iris-automl/tests/prediction/test_predictor.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_automl.prediction import Predictor
+
+
+class TestPredictor:
+ def test_execute(self, mocked_params):
+ ac = Predictor()
+ ac.execute(input_message="fake message", params=mocked_params)
+ assert not ac._params
\ No newline at end of file
diff --git a/public-engines/iris-automl/tests/training/test_metrics_evaluator.py b/public-engines/iris-automl/tests/training/test_metrics_evaluator.py
new file mode 100644
index 0000000..376bbea
--- /dev/null
+++ b/public-engines/iris-automl/tests/training/test_metrics_evaluator.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_automl.training import MetricsEvaluator
+
+
+class TestMetricsEvaluator:
+ def test_execute(self, mocked_params):
+ ac = MetricsEvaluator()
+ ac.execute(params=mocked_params)
+ assert not ac._params
\ No newline at end of file
diff --git a/public-engines/iris-automl/tests/training/test_trainer.py b/public-engines/iris-automl/tests/training/test_trainer.py
new file mode 100644
index 0000000..49ba506
--- /dev/null
+++ b/public-engines/iris-automl/tests/training/test_trainer.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_iris_automl.training import Trainer
+
+
+class TestTrainer:
+ def test_execute(self, mocked_params):
+ ac = Trainer()
+ ac.execute(params=mocked_params)
+ assert not ac._params
diff --git a/public-engines/iris-automl/tox.ini b/public-engines/iris-automl/tox.ini
new file mode 100644
index 0000000..2019932
--- /dev/null
+++ b/public-engines/iris-automl/tox.ini
@@ -0,0 +1,8 @@
+[tox]
+envlist = py27
+
+[testenv]
+deps=pytest
+ pytest-cov
+ mock
+commands=py.test --cov={envsitepackagesdir}/marvin_iris_automl --cov-report html --cov-report xml {posargs}
\ No newline at end of file
