IGNITE-12659 [ML] Remove broken sub-modules in ML, fix examples (#7430)
* IGNITE-12659 [ML] Remove broken sub-modules in ML, fix examples
* IGNITE-12659: [ML] Fix examples like in IGN-12673
diff --git a/bin/ignite-tf.sh b/bin/ignite-tf.sh
deleted file mode 100755
index af2b5d4..0000000
--- a/bin/ignite-tf.sh
+++ /dev/null
@@ -1,189 +0,0 @@
-#!/usr/bin/env bash
-if [ ! -z "${IGNITE_SCRIPT_STRICT_MODE:-}" ]
-then
- set -o nounset
- set -o errexit
- set -o pipefail
- set -o errtrace
- set -o functrace
-fi
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# Command line tool for Tensorflow cluster management.
-#
-
-#
-# Import common functions.
-#
-if [ "${IGNITE_HOME:-}" = "" ];
- then IGNITE_HOME_TMP="$(dirname "$(cd "$(dirname "$0")"; "pwd")")";
- else IGNITE_HOME_TMP=${IGNITE_HOME};
-fi
-
-#
-# Set SCRIPTS_HOME - base path to scripts.
-#
-SCRIPTS_HOME="${IGNITE_HOME_TMP}/bin"
-
-source "${SCRIPTS_HOME}"/include/functions.sh
-
-#
-# Discover path to Java executable and check it's version.
-#
-checkJava
-
-#
-# Discover IGNITE_HOME environment variable.
-#
-setIgniteHome
-
-if [ "${DEFAULT_CONFIG:-}" == "" ]; then
- DEFAULT_CONFIG=config/default-config.xml
-fi
-
-#
-# Set IGNITE_LIBS.
-#
-. "${SCRIPTS_HOME}"/include/setenv.sh
-. "${SCRIPTS_HOME}"/include/build-classpath.sh # Will be removed in the binary release.
-IGNITE_OPT_LIBS=${IGNITE_HOME}/libs/optional/
-CP="${IGNITE_LIBS}:${IGNITE_OPT_LIBS}/ignite-tensorflow/*:${IGNITE_OPT_LIBS}/ignite-slf4j/*"
-
-RANDOM_NUMBER=$("$JAVA" -cp "${CP}" org.apache.ignite.startup.cmdline.CommandLineRandomNumberGenerator)
-
-RESTART_SUCCESS_FILE="${IGNITE_HOME}/work/ignite_success_${RANDOM_NUMBER}"
-RESTART_SUCCESS_OPT="-DIGNITE_SUCCESS_FILE=${RESTART_SUCCESS_FILE}"
-
-# Mac OS specific support to display correct name in the dock.
-osname=`uname`
-
-if [ "${DOCK_OPTS:-}" == "" ]; then
- DOCK_OPTS="-Xdock:name=Ignite Node"
-fi
-
-#
-# JVM options. See http://java.sun.com/javase/technologies/hotspot/vmoptions.jsp for more details.
-#
-# ADD YOUR/CHANGE ADDITIONAL OPTIONS HERE
-#
-if [ -z "${JVM_OPTS:-}" ] ; then
- JVM_OPTS="-Xms1g -Xmx1g -server -XX:MaxMetaspaceSize=256m"
-fi
-
-#
-# Uncomment the following GC settings if you see spikes in your throughput due to Garbage Collection.
-#
-# JVM_OPTS="$JVM_OPTS -XX:+UseG1GC"
-
-#
-# Uncomment if you get StackOverflowError.
-# On 64 bit systems this value can be larger, e.g. -Xss16m
-#
-# JVM_OPTS="${JVM_OPTS} -Xss4m"
-
-#
-# Uncomment to set preference for IPv4 stack.
-#
-# JVM_OPTS="${JVM_OPTS} -Djava.net.preferIPv4Stack=true"
-
-#
-# Assertions are disabled by default since version 3.5.
-# If you want to enable them - set 'ENABLE_ASSERTIONS' flag to '1'.
-#
-ENABLE_ASSERTIONS="1"
-
-#
-# Set '-ea' options if assertions are enabled.
-#
-if [ "${ENABLE_ASSERTIONS:-}" = "1" ]; then
- JVM_OPTS="${JVM_OPTS} -ea"
-fi
-
-MAIN_CLASS=org.apache.ignite.tensorflow.submitter.JobSubmitter
-
-#
-# Remote debugging (JPDA).
-# Uncomment and change if remote debugging is required.
-#
-# JVM_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8787 ${JVM_OPTS}"
-
-#
-# Final JVM_OPTS for Java 9+ compatibility
-#
-if [ $version -eq 8 ] ; then
- JVM_OPTS="\
- -XX:+AggressiveOpts \
- ${JVM_OPTS}"
-
-elif [ $version -gt 8 ] && [ $version -lt 11 ]; then
- JVM_OPTS="\
- -XX:+AggressiveOpts \
- --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED \
- --add-exports=java.base/sun.nio.ch=ALL-UNNAMED \
- --add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED \
- --add-exports=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED \
- --add-exports=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED \
- --illegal-access=permit \
- --add-modules=java.xml.bind \
- ${JVM_OPTS}"
-
-elif [ $version -ge 11 ] ; then
- JVM_OPTS="\
- --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED \
- --add-exports=java.base/sun.nio.ch=ALL-UNNAMED \
- --add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED \
- --add-exports=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED \
- --add-exports=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED \
- --illegal-access=permit \
- ${JVM_OPTS}"
-fi
-
-
-ERRORCODE="-1"
-
-QUIET="-DIGNITE_QUIET=false"
-
-while [ "${ERRORCODE}" -ne "130" ]
-do
- case $osname in
- Darwin*)
- "$JAVA" ${JVM_OPTS:-} ${QUIET:-} "${DOCK_OPTS}" "${RESTART_SUCCESS_OPT}" \
- -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="${IGNITE_HOME}" \
- -DIGNITE_PROG_NAME="$0" -cp "${CP}" ${MAIN_CLASS} "${CONFIG:-}" "$@"
- ;;
- *)
- "$JAVA" ${JVM_OPTS:-} ${QUIET:-} "${RESTART_SUCCESS_OPT}" \
- -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="${IGNITE_HOME}" \
- -DIGNITE_PROG_NAME="$0" -cp "${CP}" ${MAIN_CLASS} "$@"
- ;;
- esac
-
- ERRORCODE="$?"
-
- if [ ! -f "${RESTART_SUCCESS_FILE}" ] ; then
- break
- else
- rm -f "${RESTART_SUCCESS_FILE}"
- fi
-done
-
-if [ -f "${RESTART_SUCCESS_FILE}" ] ; then
- rm -f "${RESTART_SUCCESS_FILE}"
-fi
diff --git a/examples/pom-standalone-lgpl.xml b/examples/pom-standalone-lgpl.xml
index a52cf48..a2f56d6 100644
--- a/examples/pom-standalone-lgpl.xml
+++ b/examples/pom-standalone-lgpl.xml
@@ -111,12 +111,6 @@
</dependency>
<dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-ml-tensorflow-model-parser</artifactId>
- <version>to_be_replaced_by_ignite_version</version>
- </dependency>
-
- <dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.2</version>
@@ -155,12 +149,6 @@
<artifactId>ignite-spark</artifactId>
<version>to_be_replaced_by_ignite_version</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-ml-mleap-model-parser</artifactId>
- <version>to_be_replaced_by_ignite_version</version>
- </dependency>
</dependencies>
<build>
@@ -248,12 +236,6 @@
<artifactId>ignite-spark-2.4</artifactId>
<version>to_be_replaced_by_ignite_version</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-ml-mleap-model-parser</artifactId>
- <version>to_be_replaced_by_ignite_version</version>
- </dependency>
</dependencies>
<build>
diff --git a/examples/pom-standalone.xml b/examples/pom-standalone.xml
index 33e7817..5c1a08d 100644
--- a/examples/pom-standalone.xml
+++ b/examples/pom-standalone.xml
@@ -111,12 +111,6 @@
</dependency>
<dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-ml-tensorflow-model-parser</artifactId>
- <version>to_be_replaced_by_ignite_version</version>
- </dependency>
-
- <dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.2</version>
@@ -155,12 +149,6 @@
<artifactId>ignite-spark</artifactId>
<version>to_be_replaced_by_ignite_version</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-ml-mleap-model-parser</artifactId>
- <version>to_be_replaced_by_ignite_version</version>
- </dependency>
</dependencies>
<build>
@@ -249,12 +237,6 @@
<artifactId>ignite-spark-2.4</artifactId>
<version>to_be_replaced_by_ignite_version</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-ml-mleap-model-parser</artifactId>
- <version>to_be_replaced_by_ignite_version</version>
- </dependency>
</dependencies>
<build>
diff --git a/examples/pom.xml b/examples/pom.xml
index 55f2d4b..3e32690 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -102,12 +102,6 @@
</dependency>
<dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-ml-tensorflow-model-parser</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.2</version>
@@ -208,12 +202,6 @@
<artifactId>ignite-spark</artifactId>
<version>${project.version}</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-ml-mleap-model-parser</artifactId>
- <version>${project.version}</version>
- </dependency>
</dependencies>
<build>
@@ -287,12 +275,6 @@
<artifactId>ignite-spark-2.4</artifactId>
<version>${project.version}</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-ml-mleap-model-parser</artifactId>
- <version>${project.version}</version>
- </dependency>
</dependencies>
<build>
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/CustomersClusterizationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/CustomersClusterizationExample.java
index ec2f3cc..5c0b8b1 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/CustomersClusterizationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/CustomersClusterizationExample.java
@@ -27,6 +27,8 @@
import org.apache.ignite.Ignition;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.lang.IgniteBiPredicate;
import org.apache.ignite.ml.clustering.kmeans.KMeansModel;
import org.apache.ignite.ml.clustering.kmeans.KMeansTrainer;
@@ -38,8 +40,6 @@
import org.apache.ignite.ml.selection.split.TrainTestDatasetSplitter;
import org.apache.ignite.ml.selection.split.TrainTestSplit;
import org.apache.ignite.ml.structures.LabeledVector;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Example of using KMeans clusterization to determine the optimal count of clusters in data.
@@ -88,7 +88,8 @@
}
}
finally {
- dataCache.destroy();
+ if (dataCache != null)
+ dataCache.destroy();
}
}
finally {
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/GmmClusterizationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/GmmClusterizationExample.java
index fa71eea..549f294 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/GmmClusterizationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/GmmClusterizationExample.java
@@ -108,7 +108,8 @@
System.out.println(">>>");
}
finally {
- dataCache.destroy();
+ if (dataCache != null)
+ dataCache.destroy();
}
}
finally {
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java
index 24b1b5c..beee4f6 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java
@@ -24,14 +24,14 @@
import org.apache.ignite.Ignition;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.clustering.kmeans.KMeansModel;
import org.apache.ignite.ml.clustering.kmeans.KMeansTrainer;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.math.Tracer;
import org.apache.ignite.ml.math.primitives.vector.Vector;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Run KMeans clustering algorithm ({@link KMeansTrainer}) over distributed dataset.
@@ -97,7 +97,8 @@
}
}
finally {
- dataCache.destroy();
+ if (dataCache != null)
+ dataCache.destroy();
}
}
finally {
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/environment/TrainingWithCustomPreprocessorsExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/environment/TrainingWithCustomPreprocessorsExample.java
index 6cd0851..878fe3c 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/environment/TrainingWithCustomPreprocessorsExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/environment/TrainingWithCustomPreprocessorsExample.java
@@ -20,6 +20,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
@@ -37,8 +39,6 @@
import org.apache.ignite.ml.selection.scoring.metric.MetricName;
import org.apache.ignite.ml.structures.LabeledVector;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* This example demonstrates an ability of using custom client classes in cluster in case of absence of these classes on
@@ -58,40 +58,49 @@
*/
public static void main(String[] args) throws Exception {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- IgniteCache<Integer, Vector> trainingSet = new SandboxMLCache(ignite)
- .fillCacheWith(MLSandboxDatasets.BOSTON_HOUSE_PRICES);
+ IgniteCache<Integer, Vector> trainingSet = null;
+ try {
+ trainingSet = new SandboxMLCache(ignite).fillCacheWith(MLSandboxDatasets.BOSTON_HOUSE_PRICES);
- Vectorizer<Integer, Vector, Integer, Double> basicVectorizer = new DummyVectorizer<Integer>()
- .labeled(Vectorizer.LabelCoordinate.FIRST);
+ Vectorizer<Integer, Vector, Integer, Double> basicVectorizer = new DummyVectorizer<Integer>()
+ .labeled(Vectorizer.LabelCoordinate.FIRST);
- Preprocessor<Integer, Vector> imputingPreprocessor = new ImputerTrainer<Integer, Vector>()
- .fit(ignite, trainingSet, basicVectorizer);
+ Preprocessor<Integer, Vector> imputingPreprocessor = new ImputerTrainer<Integer, Vector>()
+ .fit(ignite, trainingSet, basicVectorizer);
- // In-place definition of custom preprocessor by lambda expression.
- Preprocessor<Integer, Vector> customPreprocessor = (k, v) -> {
- LabeledVector res = imputingPreprocessor.apply(k, v);
- double fifthFeature = res.features().get(5);
+ // In-place definition of custom preprocessor by lambda expression.
+ Preprocessor<Integer, Vector> customPreprocessor = (k, v) -> {
+ LabeledVector res = imputingPreprocessor.apply(k, v);
+ double fifthFeature = res.features().get(5);
- Vector updatedVector = res.features().set(5, fifthFeature > 0 ? Math.log(fifthFeature) : -1);
- return updatedVector.labeled(res.label());
- };
+ Vector updatedVector = res.features().set(5, fifthFeature > 0 ? Math.log(fifthFeature) : -1);
+ return updatedVector.labeled(res.label());
+ };
- Vectorizer9000 customVectorizer = new Vectorizer9000(customPreprocessor);
+ Vectorizer9000 customVectorizer = new Vectorizer9000(customPreprocessor);
- PipelineMdl<Integer, Vector> mdl = new Pipeline<Integer, Vector, Integer, Double>()
- .addVectorizer(customVectorizer)
- .addPreprocessingTrainer(new MinMaxScalerTrainer<Integer, Vector>())
- .addPreprocessingTrainer(new NormalizationTrainer<Integer, Vector>().withP(1))
- .addPreprocessingTrainer(getCustomTrainer())
- .addTrainer(new DecisionTreeClassificationTrainer(5, 0))
- .fit(ignite, trainingSet);
+ PipelineMdl<Integer, Vector> mdl = new Pipeline<Integer, Vector, Integer, Double>()
+ .addVectorizer(customVectorizer)
+ .addPreprocessingTrainer(new MinMaxScalerTrainer<Integer, Vector>())
+ .addPreprocessingTrainer(new NormalizationTrainer<Integer, Vector>().withP(1))
+ .addPreprocessingTrainer(getCustomTrainer())
+ .addTrainer(new DecisionTreeClassificationTrainer(5, 0))
+ .fit(ignite, trainingSet);
- System.out.println(">>> Perform scoring.");
- double score = Evaluator.evaluate(trainingSet,
- mdl, mdl.getPreprocessor(), MetricName.R2
- );
+ System.out.println(">>> Perform scoring.");
+ double score = Evaluator.evaluate(trainingSet,
+ mdl, mdl.getPreprocessor(), MetricName.R2
+ );
- System.out.println(">>> R^2 score: " + score);
+ System.out.println(">>> R^2 score: " + score);
+ }
+ finally {
+ if (trainingSet != null)
+ trainingSet.destroy();
+ }
+ }
+ finally {
+ System.out.flush();
}
}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/change/Coin.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/change/Coin.java
deleted file mode 100644
index 2ce5dbb..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/change/Coin.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.change;
-
-import java.io.Serializable;
-
-/**
- * POJO to model a coin.
- */
-public class Coin implements Serializable {
- /**
- * Define Coin Type.
- */
- public enum CoinType {
- /**
- *
- */
- PENNY,
- /**
- *
- */
- QUARTER,
- /**
- *
- */
- NICKEL,
- /**
- *
- */
- DIME
- }
-
- /**
- * Number of coins.
- */
- private int numOfCoins;
-
- /**
- * Coin type.
- */
- private CoinType coinType;
-
- /**
- * Create instance.
- *
- * @param coinType Type of coin.
- * @param numOfCoins Number of coins.
- */
- Coin(CoinType coinType, int numOfCoins) {
- this.coinType = coinType;
- this.numOfCoins = numOfCoins;
- }
-
- /**
- * Retrieve the number of coins.
- *
- * @return Number of coins.
- */
- public int getNumOfCoins() {
- return numOfCoins;
- }
-
- /**
- * Set the number of coins.
- *
- * @param numOfCoins Number of coins.
- */
- public void setNumOfCoins(int numOfCoins) {
- this.numOfCoins = numOfCoins;
- }
-
- /**
- * Retrieve Coin type.
- *
- * @return Coin type.
- */
- public CoinType getCoinType() {
- return coinType;
- }
-
- /**
- * Set Coin type.
- *
- * @param coinType Coin type.
- */
- public void setCoinType(CoinType coinType) {
- this.coinType = coinType;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public String toString() {
- return "Coin [numOfCoins=" + numOfCoins + ", coinType=" + coinType + "]";
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/change/OptimizeMakeChangeFitnessFunction.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/change/OptimizeMakeChangeFitnessFunction.java
deleted file mode 100644
index 9bcc289..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/change/OptimizeMakeChangeFitnessFunction.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.change;
-
-import java.util.List;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.IFitnessFunction;
-
-/**
- * This example demonstrates how to create a {@link IFitnessFunction}.
- * <p>
- * Your fitness function will vary depending on your particular use case. For this fitness function, we simply want to
- * calculate the value of an individual solution relative to other solutions.</p>
- */
-public class OptimizeMakeChangeFitnessFunction implements IFitnessFunction {
- /**
- * Target amount.
- */
- private int targetAmount;
-
- /**
- * @param targetAmount Amount of change.
- */
- public OptimizeMakeChangeFitnessFunction(int targetAmount) {
- this.targetAmount = targetAmount;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public double evaluate(List<Gene> genes) {
- int changeAmount = getAmountOfChange(genes);
- int totalCoins = getTotalNumberOfCoins(genes);
- int changeDifference = Math.abs(targetAmount - changeAmount);
-
- double fitness = (99 - changeDifference);
-
- if (changeAmount == targetAmount)
- fitness += 100 - (10 * totalCoins);
-
- return fitness;
- }
-
- /**
- * Calculate amount of change.
- *
- * @param genes List of genes.
- * @return Amount of change.
- */
- private int getAmountOfChange(List<Gene> genes) {
- Gene quarterGene = genes.get(0);
- Gene dimeGene = genes.get(1);
- Gene nickelGene = genes.get(2);
- Gene pennyGene = genes.get(3);
-
- int numQuarters = ((Coin)quarterGene.getVal()).getNumOfCoins();
- int numDimes = ((Coin)dimeGene.getVal()).getNumOfCoins();
- int numNickels = ((Coin)nickelGene.getVal()).getNumOfCoins();
- int numPennies = ((Coin)pennyGene.getVal()).getNumOfCoins();
-
- return (numQuarters * 25) + (numDimes * 10) + (numNickels * 5) + numPennies;
- }
-
- /**
- * Return the total number of coins.
- *
- * @param genes List of genes.
- * @return Number of coins.
- */
- private int getTotalNumberOfCoins(List<Gene> genes) {
- int totalNumOfCoins = 0;
-
- for (Gene gene : genes) {
- int numOfCoins = ((Coin)gene.getVal()).getNumOfCoins();
- totalNumOfCoins += numOfCoins;
- }
-
- return totalNumOfCoins;
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/change/OptimizeMakeChangeGAExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/change/OptimizeMakeChangeGAExample.java
deleted file mode 100644
index e9415e0..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/change/OptimizeMakeChangeGAExample.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.change;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.ml.genetic.Chromosome;
-import org.apache.ignite.ml.genetic.GAGrid;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.parameter.ChromosomeCriteria;
-import org.apache.ignite.ml.genetic.parameter.GAConfiguration;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-
-/**
- * This example demonstrates how to use the {@link GAGrid} framework. It is inspired by
- * <a href="https://github.com/martin-steghoefer/jgap/blob/master/examples/src/examples/MinimizingMakeChange.java">
- * JGAP's "Minimize Make Change"</a> example.
- * <p>
- * In this example, the objective is to calculate the minimum number of coins that equal user specified amount of change
- * ie: {@code -DAMOUNTCHANGE}.</p>
- * <p>
- * {@code mvn exec:java -Dexec.mainClass="org.apache.ignite.examples.ml.genetic.change.OptimizeMakeChangeGAExample"
- * -DAMOUNTCHANGE=75}</p>
- * <p>
- * Code in this example launches Ignite grid, prepares simple test data (gene pool) and configures GA grid.</p>
- * <p>
- * After that it launches the process of evolution on GA grid and outputs the progress and results.</p>
- * <p>
- * You can change the test data and parameters of GA grid used in this example and re-run it to explore this
- * functionality further.</p>
- * <p>
- * Remote nodes should always be started with special configuration file which enables P2P class loading: {@code
- * 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
- * <p>
- * Alternatively you can run ExampleNodeStartup in another JVM which will start node with {@code
- * examples/config/example-ignite.xml} configuration.</p>
- */
-public class OptimizeMakeChangeGAExample {
- /**
- * Executes example.
- * <p>
- * Specify value for {@code -DAMOUNTCHANGE} JVM system variable.
- *
- * @param args Command line arguments, none required.
- */
- public static void main(String args[]) {
- System.out.println(">>> OptimizeMakeChange GA grid example started.");
-
- String sAmountChange = "75";
-
- StringBuilder sbErrorMsg = new StringBuilder();
- sbErrorMsg.append("AMOUNTCHANGE System property not set. Please provide a valid value between 1 and 99. ");
- sbErrorMsg.append(" ");
- sbErrorMsg.append("IE: -DAMOUNTCHANGE=75");
- sbErrorMsg.append("\n");
- sbErrorMsg.append("Using default value: 75");
-
- //Check if -DAMOUNTCHANGE JVM system variable is provided.
- if (System.getProperty("AMOUNTCHANGE") == null)
- System.out.println(sbErrorMsg);
- else
- sAmountChange = System.getProperty("AMOUNTCHANGE");
-
- try {
- // Create an Ignite instance as you would in any other use case.
- Ignite ignite = Ignition.start("examples/config/example-ignite.xml");
-
- // Create GAConfiguration.
- GAConfiguration gaCfg = new GAConfiguration();
-
- // Set Gene Pool.
- List<Gene> genes = getGenePool();
-
- // Set selection method.
- gaCfg.setSelectionMtd(GAGridConstants.SELECTION_METHOD.SELECTION_METHOD_ELITISM);
- gaCfg.setElitismCnt(10);
-
- // Set the Chromosome Length to '4' since we have 4 coins.
- gaCfg.setChromosomeLen(4);
-
- // Set population size.
- gaCfg.setPopulationSize(500);
-
- // Initialize gene pool.
- gaCfg.setGenePool(genes);
-
- // Set Truncate Rate.
- gaCfg.setTruncateRate(.10);
-
- // Set Cross Over Rate.
- gaCfg.setCrossOverRate(.50);
-
- // Set Mutation Rate.
- gaCfg.setMutationRate(.50);
-
- // Create and set Fitness function.
- OptimizeMakeChangeFitnessFunction function = new OptimizeMakeChangeFitnessFunction(new Integer(sAmountChange));
- gaCfg.setFitnessFunction(function);
-
- // Create and set TerminateCriteria.
- OptimizeMakeChangeTerminateCriteria termCriteria = new OptimizeMakeChangeTerminateCriteria(ignite,
- System.out::println);
-
- ChromosomeCriteria chromosomeCriteria = new ChromosomeCriteria();
-
- List<String> values = new ArrayList<>();
-
- values.add("coinType=QUARTER");
- values.add("coinType=DIME");
- values.add("coinType=NICKEL");
- values.add("coinType=PENNY");
-
- chromosomeCriteria.setCriteria(values);
-
- gaCfg.setChromosomeCriteria(chromosomeCriteria);
- gaCfg.setTerminateCriteria(termCriteria);
-
- // Initialize GAGrid.
- GAGrid gaGrid = new GAGrid(gaCfg, ignite);
-
- System.out.println("##########################################################################################");
-
- System.out.println("Calculating optimal set of coins where amount of change is " + sAmountChange);
-
- System.out.println("##########################################################################################");
-
- Chromosome chromosome = gaGrid.evolve();
-
- System.out.println(">>> Evolution result: " + chromosome);
-
- Ignition.stop(true);
-
- System.out.println(">>> OptimizeMakeChange GA grid example completed.");
- }
- catch (Exception e) {
- System.out.println(e.getMessage());
- e.printStackTrace();
- }
- }
-
- /**
- * Helper routine to initialize Gene pool.
- * <p>
- * In typical use case genes may be stored in database.
- *
- * @return List of Genes.
- */
- private static List<Gene> getGenePool() {
- List<Gene> list = new ArrayList<>();
-
- Gene quarterGene1 = new Gene(new Coin(Coin.CoinType.QUARTER, 3));
- Gene quarterGene2 = new Gene(new Coin(Coin.CoinType.QUARTER, 2));
- Gene quarterGene3 = new Gene(new Coin(Coin.CoinType.QUARTER, 1));
- Gene quarterGene4 = new Gene(new Coin(Coin.CoinType.QUARTER, 0));
-
- Gene dimeGene1 = new Gene(new Coin(Coin.CoinType.DIME, 2));
- Gene dimeGene2 = new Gene(new Coin(Coin.CoinType.DIME, 1));
- Gene dimeGene3 = new Gene(new Coin(Coin.CoinType.DIME, 0));
-
- Gene nickelGene1 = new Gene(new Coin(Coin.CoinType.NICKEL, 1));
- Gene nickelGene2 = new Gene(new Coin(Coin.CoinType.NICKEL, 0));
-
- Gene pennyGene1 = new Gene(new Coin(Coin.CoinType.PENNY, 4));
- Gene pennyGene2 = new Gene(new Coin(Coin.CoinType.PENNY, 3));
- Gene pennyGene3 = new Gene(new Coin(Coin.CoinType.PENNY, 2));
- Gene pennyGene4 = new Gene(new Coin(Coin.CoinType.PENNY, 1));
- Gene pennyGene5 = new Gene(new Coin(Coin.CoinType.PENNY, 0));
-
- list.add(quarterGene1);
- list.add(quarterGene2);
- list.add(quarterGene3);
- list.add(quarterGene4);
- list.add(dimeGene1);
- list.add(dimeGene2);
- list.add(dimeGene3);
- list.add(nickelGene1);
- list.add(nickelGene2);
- list.add(pennyGene1);
- list.add(pennyGene2);
- list.add(pennyGene3);
- list.add(pennyGene4);
- list.add(pennyGene5);
-
- return list;
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/change/OptimizeMakeChangeTerminateCriteria.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/change/OptimizeMakeChangeTerminateCriteria.java
deleted file mode 100644
index 267d334..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/change/OptimizeMakeChangeTerminateCriteria.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.change;
-
-import java.util.List;
-import java.util.function.Consumer;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.genetic.Chromosome;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.parameter.ITerminateCriteria;
-import org.apache.ignite.ml.genetic.utils.GAGridUtils;
-
-/**
- * Terminate Condition implementation for {@link OptimizeMakeChangeGAExample}.
- */
-public class OptimizeMakeChangeTerminateCriteria implements ITerminateCriteria {
- /**
- *
- */
- private final Ignite ignite;
-
- /**
- *
- */
- private final Consumer<String> logConsumer;
-
- /**
- * Create class instance.
- *
- * @param ignite Ignite instance.
- * @param logConsumer Logging consumer.
- */
- OptimizeMakeChangeTerminateCriteria(Ignite ignite, Consumer<String> logConsumer) {
- this.ignite = ignite;
- this.logConsumer = logConsumer;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public boolean isTerminationConditionMet(Chromosome fittestChromosome, double averageFitnessScore,
- int currGeneration) {
- boolean isTerminate = true;
-
- logConsumer.accept(
- "\n##########################################################################################"
- + "\n Generation: " + currGeneration
- + "\n Fittest is Chromosome Key: " + fittestChromosome
- + "\n Chromosome: " + fittestChromosome
- + "\n" + reportCoins(GAGridUtils.getGenesInOrderForChromosome(ignite, fittestChromosome))
- + "\nAvg Chromosome Fitness: " + averageFitnessScore
- + "\n##########################################################################################");
-
- if (!(currGeneration > 5))
- isTerminate = false;
-
- return isTerminate;
- }
-
- /**
- * Helper to print change details.
- *
- * @param genes List if Genes.
- * @return Details to print.
- */
- private String reportCoins(List<Gene> genes) {
- StringBuilder sb = new StringBuilder();
-
- for (Gene gene : genes) {
- sb.append("\nCoin Type: ").append(((Coin)gene.getVal()).getCoinType().toString())
- .append("\nNumber of Coins: ").append(((Coin)gene.getVal()).getNumOfCoins());
- }
-
- return sb.toString();
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/helloworld/HelloWorldFitnessFunction.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/helloworld/HelloWorldFitnessFunction.java
deleted file mode 100644
index 9311b78..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/helloworld/HelloWorldFitnessFunction.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.helloworld;
-
-import java.util.List;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.IFitnessFunction;
-
-/**
- * This example demonstrates how to create a {@link IFitnessFunction}.
- * <p>
- * Your fitness function will vary depending on your particular use case. For this fitness function, we simply want to
- * calculate the value of an individual solution relative to other solutions.
- * <p>
- * To do this, we increase fitness score by '1' for each character that is in correct position.</p>
- * <p>
- * For our solution, genetic algorithm will continue until we achieve a fitness score of '11', as 'HELLO WORLD' contains
- * 11 characters.</p>
- */
-public class HelloWorldFitnessFunction implements IFitnessFunction {
- /**
- * {@inheritDoc}
- */
- @Override public double evaluate(List<Gene> genes) {
- double matches = 0;
-
- for (int i = 0; i < genes.size(); i++) {
- String targetStr = "HELLO WORLD";
- if (genes.get(i).getVal().equals(targetStr.charAt(i)))
- matches += 1;
- }
- return matches;
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/helloworld/HelloWorldGAExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/helloworld/HelloWorldGAExample.java
deleted file mode 100644
index 5a9ac62..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/helloworld/HelloWorldGAExample.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.helloworld;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.ml.genetic.Chromosome;
-import org.apache.ignite.ml.genetic.GAGrid;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.parameter.GAConfiguration;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-
-/**
- * This example demonstrates how to use the {@link GAGrid} framework. In this example, we want to evolve a string of 11
- * characters such that the word 'HELLO WORLD' is found.
- * <p>
- * Code in this example launches Ignite grid, prepares simple test data (gene pool) and configures GA grid.</p>
- * <p>
- * After that it launches the process of evolution on GA grid and outputs the progress and results.</p>
- * <p>
- * You can change the test data and parameters of GA grid used in this example and re-run it to explore this
- * functionality further.</p>
- * <p>
- * For example, you may change the some basic genetic parameters on the GAConfiguration object:
- * <p>
- * Mutation Rate Crossover Rate Population Size Selection Method
- *
- * <p>
- * How to run from command line:</p>
- * <p>
- * {@code mvn exec:java -Dexec.mainClass="org.apache.ignite.examples.ml.genetic.helloworld.HelloWorldGAExample"}</p>
- * <p>
- * Remote nodes should always be started with special configuration file which enables P2P class loading: {@code
- * 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
- * <p>
- * Alternatively you can run ExampleNodeStartup in another JVM which will start node with {@code
- * examples/config/example-ignite.xml} configuration.</p>
- */
-public class HelloWorldGAExample {
- /**
- * Executes example.
- *
- * @param args Command line arguments, none required.
- */
- public static void main(String args[]) {
- System.out.println(">>> HelloWorld GA grid example started.");
-
- try {
- // Create an Ignite instance as you would in any other use case.
- Ignite ignite = Ignition.start("examples/config/example-ignite.xml");
-
- // Create GAConfiguration.
- GAConfiguration gaCfg = new GAConfiguration();
-
- // Set Gene Pool.
- List<Gene> genes = getGenePool();
-
- // Set the Chromosome Length to '11' since 'HELLO WORLD' contains 11 characters.
- gaCfg.setChromosomeLen(11);
-
- // Initialize gene pool.
- gaCfg.setGenePool(genes);
-
- // Set CrossOver Rate.
- gaCfg.setCrossOverRate(.05);
-
- // Set Mutation Rate.
- gaCfg.setMutationRate(.05);
-
- // Set Selection Method.
- gaCfg.setSelectionMtd(GAGridConstants.SELECTION_METHOD.SELECTION_METHOD_ROULETTE_WHEEL);
-
- // Set Population Size.
- gaCfg.setPopulationSize(2000);
-
- // Create and set Fitness function.
- HelloWorldFitnessFunction function = new HelloWorldFitnessFunction();
- gaCfg.setFitnessFunction(function);
-
- // Create and set TerminateCriteria.
- AtomicInteger cnt = new AtomicInteger(0);
- HelloWorldTerminateCriteria termCriteria = new HelloWorldTerminateCriteria(ignite,
- msg -> {
- if (cnt.getAndIncrement() % 20 == 0)
- System.out.println(msg);
- });
-
- gaCfg.setTerminateCriteria(termCriteria);
-
- GAGrid gaGrid = new GAGrid(gaCfg, ignite);
-
- // Evolve the population.
- Chromosome chromosome = gaGrid.evolve();
-
- System.out.println(">>> Evolution result: " + chromosome);
-
- Ignition.stop(true);
-
- System.out.println(">>> HelloWorld GA grid example completed.");
- }
- catch (Exception e) {
- System.out.println(e.getMessage());
- e.printStackTrace();
- }
- }
-
- /**
- * Helper routine to initialize Gene pool.
- * <p>
- * In typical use case genes may be stored in database.
- *
- * @return List of Gene objects.
- */
- private static List<Gene> getGenePool() {
- List<Gene> list = new ArrayList<>();
-
- char[] chars = {
- 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S',
- 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', ' '};
-
- for (char aChar : chars) {
- Gene gene = new Gene(aChar);
- list.add(gene);
- }
-
- return list;
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/helloworld/HelloWorldTerminateCriteria.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/helloworld/HelloWorldTerminateCriteria.java
deleted file mode 100644
index 66a8374..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/helloworld/HelloWorldTerminateCriteria.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.helloworld;
-
-import java.util.List;
-import java.util.function.Consumer;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.genetic.Chromosome;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.parameter.ITerminateCriteria;
-import org.apache.ignite.ml.genetic.utils.GAGridUtils;
-
-/**
- * Represents the terminate condition for {@link HelloWorldGAExample}.
- * <p>
- * Class terminates Genetic algorithm when fitness score is more than 10.</p>
- */
-public class HelloWorldTerminateCriteria implements ITerminateCriteria {
- /**
- * Ignite instance.
- */
- private final Ignite ignite;
-
- /**
- *
- */
- private final Consumer<String> logConsumer;
-
- /**
- * Create class instance.
- *
- * @param ignite Ignite instance.
- * @param logConsumer Logging consumer.
- */
- HelloWorldTerminateCriteria(Ignite ignite, Consumer<String> logConsumer) {
- this.ignite = ignite;
- this.logConsumer = logConsumer;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public boolean isTerminationConditionMet(Chromosome fittestChromosome, double averageFitnessScore,
- int currGeneration) {
- boolean isTerminate = true;
-
- logConsumer.accept(
- "\n##########################################################################################"
- + "\n Generation: " + currGeneration
- + "\n Fittest is Chromosome Key: " + fittestChromosome
- + "\n Chromosome: " + fittestChromosome
- + "\n" + printPhrase(GAGridUtils.getGenesInOrderForChromosome(ignite, fittestChromosome))
- + "\nAvg Chromosome Fitness: " + averageFitnessScore
- + "\n##########################################################################################");
-
- if (!(fittestChromosome.getFitnessScore() > 10))
- isTerminate = false;
-
- return isTerminate;
- }
-
- /**
- * Helper to print phrase.
- *
- * @param genes List of Genes.
- * @return Phrase to print.
- */
- private String printPhrase(List<Gene> genes) {
- StringBuilder sbPhrase = new StringBuilder();
-
- for (Gene gene : genes)
- sbPhrase.append(((Character)gene.getVal()).toString());
-
- return sbPhrase.toString();
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/knapsack/Item.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/knapsack/Item.java
deleted file mode 100644
index 6bf7947..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/knapsack/Item.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.knapsack;
-
-import java.io.Serializable;
-
-/**
- * POJO to model an Item.
- */
-public class Item implements Serializable {
- /**
- * Weight of item in lbs.
- */
- private double weight;
-
- /**
- * Value of item.
- */
- private double val;
-
- /**
- * Name of item.
- */
- private String name;
-
- /**
- * Get the weight.
- *
- * @return Weight.
- */
- public double getWeight() {
- return weight;
- }
-
- /**
- * Set the weight.
- *
- * @param weight Weight.
- */
- public void setWeight(double weight) {
- this.weight = weight;
- }
-
- /**
- * Get the value.
- *
- * @return Value.
- */
- public double getVal() {
- return val;
- }
-
- /**
- * Set the value.
- *
- * @param val Value.
- */
- public void setVal(double val) {
- this.val = val;
- }
-
- /**
- * Get the name.
- *
- * @return Name
- */
- public String getName() {
- return name;
- }
-
- /**
- * Set the name.
- *
- * @param name Name.
- */
- public void setName(String name) {
- this.name = name;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public String toString() {
- return "Item [weight=" + weight + ", value=" + val + ", name=" + name + "]";
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/knapsack/KnapsackFitnessFunction.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/knapsack/KnapsackFitnessFunction.java
deleted file mode 100644
index fe3e24d..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/knapsack/KnapsackFitnessFunction.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.knapsack;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.IFitnessFunction;
-
-/**
- * This example demonstrates how to create a {@link IFitnessFunction}.
- * <p>
- * Your fitness function will vary depending on your particular use case. For this fitness function, we simply want to
- * calculate the weight and value of an individual solution relative to other solutions.</p>
- * <p>
- * To do this, we total the weights and values of all the genes within a chromosome.</p>
- */
-public class KnapsackFitnessFunction implements IFitnessFunction {
- /**
- * {@inheritDoc}
- */
- @Override public double evaluate(List<Gene> genes) {
- double val = 0;
- double weight = 0;
-
- List<Long> duplicates = new ArrayList<>();
- int badSolution = 1;
-
- for (Gene gene : genes) {
- weight += ((Item)(gene.getVal())).getWeight();
- val += ((Item)(gene.getVal())).getVal();
-
- double maximumWeight = 20;
- if (duplicates.contains(gene.id()) || (weight > maximumWeight)) {
- badSolution = 0;
- break;
- }
- else
- duplicates.add(gene.id());
- }
-
- return (val * badSolution);
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/knapsack/KnapsackGAExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/knapsack/KnapsackGAExample.java
deleted file mode 100644
index cd3cdfd4..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/knapsack/KnapsackGAExample.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.knapsack;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.ml.genetic.Chromosome;
-import org.apache.ignite.ml.genetic.GAGrid;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.parameter.GAConfiguration;
-
-/**
- * This example demonstrates how to use the {@link GAGrid} framework. It shows working with Knapsack Problem: Given a
- * set of 30 items, each with a weight and a value, pack 10 items in knapsack so that the total weight is less or equal
- * 20 lbs, and the total value is maximized.
- * <p>
- * Code in this example launches Ignite grid, prepares simple test data (gene pool) and configures GA grid.</p>
- * <p>
- * After that it launches the process of evolution on GA grid and outputs the progress and results.</p>
- * <p>
- * You can change the test data and parameters of GA grid used in this example and re-run it to explore this
- * functionality further.</p>
- * <p>
- * How to run from command line:</p>
- * <p>
- * {@code mvn exec:java -Dexec.mainClass="org.apache.ignite.examples.ml.genetic.knapsack.KnapsackGAExample"}</p>
- * <p>
- * Remote nodes should always be started with special configuration file which enables P2P class loading: {@code
- * 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
- * <p>
- * Alternatively you can run ExampleNodeStartup in another JVM which will start node with {@code
- * examples/config/example-ignite.xml} configuration.</p>
- */
-public class KnapsackGAExample {
- /**
- * @param args Command line arguments, none required.
- */
- public static void main(String args[]) {
- System.out.println(">>> Knapsack GA grid example started.");
-
- try {
- // Create an Ignite instance as you would in any other use case.
- Ignite ignite = Ignition.start("examples/config/example-ignite.xml");
-
- // Create GAConfiguration.
- GAConfiguration gaCfg = new GAConfiguration();
-
- // Set Gene Pool.
- List<Gene> genes = getGenePool();
-
- // Set the Chromosome Length to '10' since our knapsack may contain a total of 10 items.
- gaCfg.setChromosomeLen(10);
-
- // Initialize gene pool.
- gaCfg.setGenePool(genes);
-
- // Create and set Fitness function.
- KnapsackFitnessFunction function = new KnapsackFitnessFunction();
- gaCfg.setFitnessFunction(function);
-
- // Create and set TerminateCriteria.
- AtomicInteger cnt = new AtomicInteger(0);
- KnapsackTerminateCriteria termCriteria = new KnapsackTerminateCriteria(ignite,
- msg -> {
- if (cnt.getAndIncrement() % 10 == 0)
- System.out.println(msg);
- });
- gaCfg.setTerminateCriteria(termCriteria);
-
- GAGrid gaGrid = new GAGrid(gaCfg, ignite);
-
- // Evolve the population.
- Chromosome chromosome = gaGrid.evolve();
-
- System.out.println(">>> Evolution result: " + chromosome);
-
- Ignition.stop(true);
-
- System.out.println(">>> Knapsack GA grid example completed.");
- }
- catch (Exception e) {
- System.out.println(e.getMessage());
- e.printStackTrace();
- }
- }
-
- /**
- * Helper routine to initialize Gene pool.
- * <p>
- * In typical use case genes may be stored in database.
- *
- * @return List of Gene objects.
- */
- private static List<Gene> getGenePool() {
- List<Gene> list = new ArrayList<>();
-
- Item item1 = new Item();
- item1.setName("Swiss Army Knife");
- item1.setWeight(0.08125);
- item1.setVal(15);
- Gene gene1 = new Gene(item1);
-
- Item item2 = new Item();
- item2.setName("Duct Tape");
- item2.setWeight(1.3);
- item2.setVal(3);
- Gene gene2 = new Gene(item2);
-
- Item item3 = new Item();
- item3.setName("Rope (50 feet)");
- item3.setWeight(7);
- item3.setVal(10);
- Gene gene3 = new Gene(item3);
-
- Item item4 = new Item();
- item4.setName("Satellite phone");
- item4.setWeight(2);
- item4.setVal(8);
- Gene gene4 = new Gene(item4);
-
- Item item5 = new Item();
- item5.setName("Elmer's Glue");
- item5.setWeight(0.25);
- item5.setVal(2);
- Gene gene5 = new Gene(item5);
-
- Item item6 = new Item();
- item6.setName("Toilet Paper Roll");
- item6.setWeight(.5);
- item6.setVal(4);
- Gene gene6 = new Gene(item6);
-
- Item item7 = new Item();
- item7.setName("Binoculars");
- item7.setWeight(3);
- item7.setVal(5);
- Gene gene7 = new Gene(item7);
-
- Item item8 = new Item();
- item8.setName("Compass");
- item8.setWeight(0.0573202);
- item8.setVal(15);
- Gene gene8 = new Gene(item8);
-
- Item item9 = new Item();
- item9.setName("Jug (pre-filled with water)");
- item9.setWeight(4);
- item9.setVal(6);
- Gene gene9 = new Gene(item9);
-
- Item item10 = new Item();
- item10.setName("Flashlight");
- item10.setWeight(2);
- item10.setVal(4);
- Gene gene10 = new Gene(item10);
-
- Item item11 = new Item();
- item11.setName("Box of paper clips");
- item11.setWeight(.9);
- item11.setVal(2);
- Gene gene11 = new Gene(item11);
-
- Item item12 = new Item();
- item12.setName("Gloves (1 pair)");
- item12.setWeight(.8125);
- item12.setVal(3);
- Gene gene12 = new Gene(item12);
-
- Item item13 = new Item();
- item13.setName("Scissors");
- item13.setWeight(0.2);
- item13.setVal(2);
- Gene gene13 = new Gene(item13);
-
- Item item14 = new Item();
- item14.setName("Signal Flair (4pk)");
- item14.setWeight(4);
- item14.setVal(5);
- Gene gene14 = new Gene(item14);
-
- Item item15 = new Item();
- item15.setName("Water Purifying System");
- item15.setWeight(0.5125);
- item15.setVal(4);
- Gene gene15 = new Gene(item15);
-
- Item item16 = new Item();
- item16.setName("Whistle");
- item16.setWeight(0.075);
- item16.setVal(2);
- Gene gene16 = new Gene(item16);
-
- Item item17 = new Item();
- item17.setName("Sleeping Bag");
- item17.setWeight(0.38125);
- item17.setVal(4);
- Gene gene17 = new Gene(item17);
-
- Item item18 = new Item();
- item18.setName("Insect Repellent");
- item18.setWeight(1.15);
- item18.setVal(3);
- Gene gene18 = new Gene(item18);
-
- Item item19 = new Item();
- item19.setName("Trowel");
- item19.setWeight(0.31875);
- item19.setVal(3);
- Gene gene19 = new Gene(item19);
-
- Item item20 = new Item();
- item20.setName("Lighter");
- item20.setWeight(.2);
- item20.setVal(4);
- Gene gene20 = new Gene(item20);
-
- Item item21 = new Item();
- item21.setName("Safety Horn");
- item21.setWeight(.21);
- item21.setVal(3);
- Gene gene21 = new Gene(item21);
-
- Item item22 = new Item();
- item22.setName("Headlamp");
- item22.setWeight(.8);
- item22.setVal(4);
- Gene gene22 = new Gene(item22);
-
- Item item23 = new Item();
- item23.setName("Freeze Dried Food Kit");
- item23.setWeight(2);
- item23.setVal(6);
- Gene gene23 = new Gene(item23);
-
- Item item24 = new Item();
- item24.setName("Sunscreen");
- item24.setWeight(.5);
- item24.setVal(4);
- Gene gene24 = new Gene(item24);
-
- Item item25 = new Item();
- item25.setName("Trekking Pole (Adjustable)");
- item25.setWeight(1.3);
- item25.setVal(4);
- Gene gene25 = new Gene(item25);
-
- Item item26 = new Item();
- item26.setName("Counter Assault Bear Spray");
- item26.setWeight(.5);
- item26.setVal(4);
- Gene gene26 = new Gene(item26);
-
- Item item27 = new Item();
- item27.setName("Insect Spray");
- item27.setWeight(.5);
- item27.setVal(3);
- Gene gene27 = new Gene(item27);
-
- Item item28 = new Item();
- item28.setName("Hand sanitizer");
- item28.setWeight(.625);
- item28.setVal(3);
- Gene gene28 = new Gene(item28);
-
- Item item29 = new Item();
- item29.setName("Mirror");
- item29.setWeight(.5);
- item29.setVal(3);
- Gene gene29 = new Gene(item29);
-
- Item item30 = new Item();
- item30.setName("First Aid Kit");
- item30.setWeight(3);
- item30.setVal(6);
- Gene gene30 = new Gene(item30);
-
- list.add(gene1);
- list.add(gene2);
- list.add(gene3);
- list.add(gene4);
- list.add(gene5);
- list.add(gene6);
- list.add(gene7);
- list.add(gene8);
- list.add(gene9);
- list.add(gene10);
- list.add(gene11);
- list.add(gene12);
- list.add(gene13);
- list.add(gene14);
- list.add(gene15);
- list.add(gene16);
- list.add(gene17);
- list.add(gene18);
- list.add(gene19);
- list.add(gene20);
- list.add(gene21);
- list.add(gene22);
- list.add(gene23);
- list.add(gene24);
- list.add(gene25);
- list.add(gene26);
- list.add(gene27);
- list.add(gene28);
- list.add(gene29);
- list.add(gene30);
-
- return list;
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/knapsack/KnapsackTerminateCriteria.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/knapsack/KnapsackTerminateCriteria.java
deleted file mode 100644
index 9bbeca5..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/knapsack/KnapsackTerminateCriteria.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.knapsack;
-
-import java.util.List;
-import java.util.function.Consumer;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.genetic.Chromosome;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.parameter.ITerminateCriteria;
-import org.apache.ignite.ml.genetic.utils.GAGridUtils;
-
-/**
- * Represents the terminate condition for {@link KnapsackGAExample}.
- * <p>
- * Class terminates Genetic algorithm when once GA Grid has performed 30 generations.</p>
- */
-public class KnapsackTerminateCriteria implements ITerminateCriteria {
- /**
- * Ignite instance.
- */
- private final Ignite ignite;
-
- /**
- *
- */
- private final Consumer<String> logConsumer;
-
- /**
- * Create class instance.
- *
- * @param ignite Ignite instance.
- * @param logConsumer Logging consumer.
- */
- KnapsackTerminateCriteria(Ignite ignite, Consumer<String> logConsumer) {
- this.ignite = ignite;
- this.logConsumer = logConsumer;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public boolean isTerminationConditionMet(Chromosome fittestChromosome, double averageFitnessScore,
- int currGeneration) {
- boolean isTerminate = true;
-
- logConsumer.accept(
- "\n##########################################################################################"
- + "\n Generation: " + currGeneration
- + "\n Fittest is Chromosome Key: " + fittestChromosome
- + "\nTotal value is: " + fittestChromosome.getFitnessScore()
- + "\nTotal weight is: " + calculateTotalWeight(
- GAGridUtils.getGenesInOrderForChromosome(ignite, fittestChromosome))
- + "\nChromosome: " + fittestChromosome
- + "\n" + reportItems(GAGridUtils.getGenesInOrderForChromosome(ignite, fittestChromosome))
- + "\n##########################################################################################");
-
- if (!(currGeneration > 29))
- isTerminate = false;
-
- return isTerminate;
- }
-
- /**
- * Calculate total weight.
- *
- * @param genes List of Genes.
- * @return Calculated value.
- */
- private double calculateTotalWeight(List<Gene> genes) {
- double totalWeight = 0;
- for (Gene gene : genes)
- totalWeight += ((Item)gene.getVal()).getWeight();
-
- return totalWeight;
- }
-
- /**
- * Helper to print items in knapsack.
- *
- * @param genes List of Genes.
- * @return Items to print.
- */
- private String reportItems(List<Gene> genes) {
- StringBuilder sb = new StringBuilder();
-
- for (Gene gene : genes) {
- sb.append("\n------------------------------------------------------------------------------------------")
- .append("\nName: ").append(((Item)gene.getVal()).getName())
- .append("\nWeight: ").append(((Item)gene.getVal()).getWeight())
- .append("\nValue: ").append(((Item)gene.getVal()).getVal());
- }
-
- return sb.toString();
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/movie/Movie.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/movie/Movie.java
deleted file mode 100644
index 2aa96d6..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/movie/Movie.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.movie;
-
-import java.util.List;
-
-/**
- * POJO to model a movie.
- */
-public class Movie {
- /**
- * Name of movie.
- */
- private String name;
-
- /**
- * Genre of movie.
- */
- private List genre;
-
- /**
- * Rating of movie.
- */
- private String rating;
-
- /**
- * IMDB rating.
- */
- private double imdbRating;
-
- /**
- * Year of movie.
- */
- private String year;
-
- /**
- * Get the year.
- *
- * @return Year.
- */
- public String getYear() {
- return year;
- }
-
- /**
- * Set the year.
- *
- * @param year Year.
- */
- public void setYear(String year) {
- this.year = year;
- }
-
- /**
- * Get the <a href="https://en.wikipedia.org/wiki/IMDb">IMDB rating</a>.
- *
- * @return IMDB rating.
- */
- public double getImdbRating() {
- return imdbRating;
- }
-
- /**
- * Set the IMDB rating.
- *
- * @param imdbRating IMDB rating.
- */
- public void setImdbRating(double imdbRating) {
- this.imdbRating = imdbRating;
- }
-
- /**
- * Get the name of movie.
- *
- * @return Name of movie.
- */
- public String getName() {
- return name;
- }
-
- /**
- * Set the name of movie.
- *
- * @param name Movie name.
- */
- public void setName(String name) {
- this.name = name;
- }
-
- /**
- * Get movie genres.
- *
- * @return List of genres.
- */
- public List getGenre() {
- return genre;
- }
-
- /**
- * Set the genre.
- *
- * @param genre List of genres of movie.
- */
- public void setGenre(List genre) {
- this.genre = genre;
- }
-
- /**
- * Get the rating of the movie.
- *
- * @return Movie rating.
- */
- public String getRating() {
- return rating;
- }
-
- /**
- * Set the rating of the movie.
- *
- * @param rating Movie rating.
- */
- public void setRating(String rating) {
- this.rating = rating;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public String toString() {
- return "Movie [name=" + name + ", genre=" + genre + ", rating=" + rating + ", imdbRating=" + imdbRating
- + ", year=" + year + "]";
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/movie/MovieFitnessFunction.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/movie/MovieFitnessFunction.java
deleted file mode 100644
index 197f3b6..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/movie/MovieFitnessFunction.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.movie;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.IFitnessFunction;
-
-/**
- * This example demonstrates how to create a {@link IFitnessFunction}.
- * <p>
- * Your fitness function will vary depending on your particular use case. For this fitness function, we want to
- * calculate the value of an individual solution relative to other solutions.</p>
- * <p>
- * To do this, we increase fitness score by number of times genre is found in list of movies. In addition, we increase
- * score by fictional IMDB rating.</p>
- * <p>
- * If there are duplicate movies in selection, we automatically apply a '0' fitness score.</p>
- */
-public class MovieFitnessFunction implements IFitnessFunction {
- /**
- * Genres.
- */
- private List<String> genres;
-
- /**
- * Create instance.
- *
- * @param genres List of genres.
- */
- public MovieFitnessFunction(List<String> genres) {
- this.genres = genres;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public double evaluate(List<Gene> genes) {
- double score = 0;
- List<String> duplicates = new ArrayList<>();
- int badSolution = 1;
-
- for (Gene gene : genes) {
- Movie movie = (Movie)gene.getVal();
- if (duplicates.contains(movie.getName()))
- badSolution = 0;
- else
- duplicates.add(movie.getName());
-
- double genreScore = getGenreScore(movie);
- if (genreScore == 0)
- badSolution = 0;
-
- score = (score + movie.getImdbRating()) + (genreScore);
- }
- return (score * badSolution);
- }
-
- /**
- * Helper to calculate genre score.
- *
- * @param movie Movie.
- * @return Genre score.
- */
- private double getGenreScore(Movie movie) {
- double genreScore = 0;
-
- for (String genre : genres) {
- if (movie.getGenre().contains(genre))
- genreScore += 1;
- }
- return genreScore;
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/movie/MovieGAExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/movie/MovieGAExample.java
deleted file mode 100644
index 6fb60c9..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/movie/MovieGAExample.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.movie;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.StringTokenizer;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.ml.genetic.Chromosome;
-import org.apache.ignite.ml.genetic.GAGrid;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.parameter.GAConfiguration;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-
-/**
- * In this example, we utilize {@link GAGrid} framework to calculate an optimal set of movies based on our interests in
- * various genres (ie: Action, Comedy, and Romance).
- * <p>
- * Code in this example launches Ignite grid, prepares simple test data (gene pool) and configures GA grid.</p>
- * <p>
- * After that it launches the process of evolution on GA grid and outputs the progress and results.</p>
- * <p>
- * You can change the test data and parameters of GA grid used in this example and re-run it to explore this
- * functionality further.</p>
- * <p>
- * How to run from command line:</p>
- * <p>
- * {@code mvn exec:java -Dexec.mainClass="org.apache.ignite.examples.ml.genetic.movie.MovieGAExample"
- * -DGENRES=Action,Comedy}</p>
- * <p>
- * Remote nodes should always be started with special configuration file which enables P2P class loading: {@code
- * 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
- * <p>
- * Alternatively you can run ExampleNodeStartup in another JVM which will start node with {@code
- * examples/config/example-ignite.xml} configuration.</p>
- */
-public class MovieGAExample {
- /**
- * Executes example.
- * <p>
- * Specify value for {@code -DGENRES} JVM system variable.
- *
- * @param args Command line arguments, none required.
- */
- public static void main(String args[]) {
- System.out.println(">>> Movie GA grid example started.");
-
- List<String> genres = new ArrayList<>();
- String sGenres = "Action,Comedy,Romance";
-
- StringBuffer sbErrorMsg = new StringBuffer();
- sbErrorMsg.append("GENRES System property not set. Please provide GENRES information.");
- sbErrorMsg.append(" ");
- sbErrorMsg.append("IE: -DGENRES=Action,Comedy,Romance");
- sbErrorMsg.append("\n");
- sbErrorMsg.append("Using default value: Action,Comedy,Romance");
-
- if (System.getProperty("GENRES") == null)
- System.out.println(sbErrorMsg);
- else
- sGenres = System.getProperty("GENRES");
-
- StringTokenizer st = new StringTokenizer(sGenres, ",");
-
- while (st.hasMoreElements()) {
- String genre = st.nextToken();
- genres.add(genre);
- }
-
- // Create GAConfiguration.
- GAConfiguration gaCfg = new GAConfiguration();
-
- // Set Gene Pool.
- List<Gene> genes = getGenePool();
-
- // Define Chromosome.
- gaCfg.setChromosomeLen(3);
- gaCfg.setPopulationSize(100);
- gaCfg.setGenePool(genes);
- gaCfg.setTruncateRate(.10);
- gaCfg.setCrossOverRate(.50);
- gaCfg.setMutationRate(.50);
- gaCfg.setSelectionMtd(GAGridConstants.SELECTION_METHOD.SELECTION_METHOD_TRUNCATION);
-
- // Create fitness function.
- MovieFitnessFunction function = new MovieFitnessFunction(genres);
-
- // Set fitness function.
- gaCfg.setFitnessFunction(function);
-
- try {
- // Create an Ignite instance as you would in any other use case.
- Ignite ignite = Ignition.start("examples/config/example-ignite.xml");
-
- MovieTerminateCriteria termCriteria = new MovieTerminateCriteria(ignite, System.out::println);
-
- gaCfg.setTerminateCriteria(termCriteria);
-
- GAGrid gaGrid = new GAGrid(gaCfg, ignite);
-
- Chromosome chromosome = gaGrid.evolve();
-
- System.out.println(">>> Evolution result: " + chromosome);
-
- Ignition.stop(true);
-
- System.out.println(">>> Movie GA grid example completed.");
- }
- catch (Exception e) {
- System.out.println(e.getMessage());
- e.printStackTrace();
- }
- }
-
- /**
- *
- */
- private static List<Gene> getGenePool() {
- List<Gene> list = new ArrayList<>();
-
- Movie movie1 = new Movie();
- movie1.setName("The Matrix");
- movie1.setImdbRating(7);
- List<String> genre1 = new ArrayList<>();
- genre1.add("SciFi");
- genre1.add("Action");
- movie1.setGenre(genre1);
- movie1.setRating("PG-13");
- movie1.setYear("1999");
-
- Gene gene1 = new Gene(movie1);
-
- Movie movie2 = new Movie();
- movie2.setName("The Dark Knight");
- movie2.setImdbRating(9.6);
- List<String> genre2 = new ArrayList<>();
- genre2.add("Action");
- movie2.setGenre(genre2);
- movie2.setRating("PG-13");
- movie2.setYear("2008");
-
- Gene gene2 = new Gene(movie2);
-
- Movie movie3 = new Movie();
- movie3.setName("The Avengers");
- movie3.setImdbRating(9.6);
- movie3.setYear("2012");
-
- List<String> genre3 = new ArrayList<>();
- genre3.add("Action");
- movie3.setGenre(genre3);
- movie3.setRating("PG-13");
-
- Gene gene3 = new Gene(movie3);
-
- Movie movie4 = new Movie();
- movie4.setName("The Hangover");
- movie4.setImdbRating(7.6);
- List<String> genre4 = new ArrayList<>();
- genre4.add("Comedy");
- movie4.setGenre(genre4);
- movie4.setRating("R");
- movie4.setYear("2009");
-
- Gene gene4 = new Gene(movie4);
-
- Movie movie5 = new Movie();
- movie5.setName("The Hangover 2");
- movie5.setImdbRating(9.6);
- List<String> genre5 = new ArrayList<>();
- genre5.add("Comedy");
- movie5.setGenre(genre5);
- movie5.setRating("R");
- movie5.setYear("2012");
-
- Gene gene5 = new Gene(movie5);
-
- Movie movie6 = new Movie();
- movie6.setName("This Means War");
- movie6.setImdbRating(6.4);
- List<String> genre6 = new ArrayList<>();
- genre6.add("Comedy");
- genre6.add("Action");
- genre6.add("Romance");
- movie6.setGenre(genre6);
- movie6.setRating("PG-13");
- movie6.setYear("2012");
-
- Gene gene6 = new Gene(movie6);
-
- Movie movie7 = new Movie();
- movie7.setName("Hitch");
- movie7.setImdbRating(10);
- List<String> genre7 = new ArrayList<>();
- genre7.add("Comedy");
- genre7.add("Romance");
- movie7.setGenre(genre7);
- movie7.setRating("PG-13");
- movie7.setYear("2005");
-
- Gene gene7 = new Gene(movie7);
-
- Movie movie8 = new Movie();
- movie8.setName("21 Jump Street");
- movie8.setImdbRating(6.7);
- List<String> genre8 = new ArrayList<>();
- genre8.add("Comedy");
- genre8.add("Action");
- movie8.setGenre(genre8);
- movie8.setRating("R");
- movie8.setYear("2012");
-
- Gene gene8 = new Gene(movie8);
-
- Movie movie9 = new Movie();
- movie9.setName("Killers");
- movie9.setImdbRating(5.1);
- List<String> genre9 = new ArrayList<>();
- genre9.add("Comedy");
- genre9.add("Action");
- genre9.add("Romance");
- movie9.setGenre(genre9);
- movie9.setRating("PG-13");
- movie9.setYear("2010");
-
- Gene gene9 = new Gene(movie9);
-
- Movie movie10 = new Movie();
- movie10.setName("What to Expect When You're Expecting");
- movie10.setImdbRating(5.1);
- List<String> genre10 = new ArrayList<>();
- genre10.add("Comedy");
- genre10.add("Romance");
- movie10.setGenre(genre10);
- movie10.setRating("PG-13");
- movie10.setYear("2012");
-
- Gene gene10 = new Gene(movie10);
-
- list.add(gene1);
- list.add(gene2);
- list.add(gene3);
- list.add(gene4);
- list.add(gene5);
- list.add(gene6);
- list.add(gene7);
- list.add(gene8);
- list.add(gene9);
- list.add(gene10);
-
- return list;
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/movie/MovieTerminateCriteria.java b/examples/src/main/java/org/apache/ignite/examples/ml/genetic/movie/MovieTerminateCriteria.java
deleted file mode 100644
index b193b7d..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/genetic/movie/MovieTerminateCriteria.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.genetic.movie;
-
-import java.util.List;
-import java.util.function.Consumer;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.genetic.Chromosome;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.parameter.ITerminateCriteria;
-import org.apache.ignite.ml.genetic.utils.GAGridUtils;
-
-/**
- * Represents the terminate condition for {@link MovieGAExample}.
- * <p>
- * Class terminates Genetic algorithm when fitness score is more than 32.</p>
- */
-public class MovieTerminateCriteria implements ITerminateCriteria {
- /**
- * Ignite instance.
- */
- private final Ignite ignite;
-
- /**
- *
- */
- private final Consumer<String> logConsumer;
-
- /**
- * Create class instance.
- *
- * @param ignite Ignite instance.
- * @param logConsumer Logging consumer.
- */
- MovieTerminateCriteria(Ignite ignite, Consumer<String> logConsumer) {
- this.ignite = ignite;
- this.logConsumer = logConsumer;
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public boolean isTerminationConditionMet(Chromosome fittestChromosome, double averageFitnessScore,
- int currGeneration) {
- boolean isTerminate = true;
-
- logConsumer.accept(
- "\n##########################################################################################"
- + "\n Generation: " + currGeneration
- + "\n Fittest is Chromosome Key: " + fittestChromosome
- + "\nChromosome: " + fittestChromosome
- + "\n" + reportMovies(GAGridUtils.getGenesInOrderForChromosome(ignite, fittestChromosome))
- + "\n##########################################################################################");
-
- if (!(fittestChromosome.getFitnessScore() > 32))
- isTerminate = false;
-
- return isTerminate;
- }
-
- /**
- * Helper to print movies details.
- *
- * @param genes List of Genes.
- * @return Movies details.
- */
- private String reportMovies(List<Gene> genes) {
- StringBuilder sb = new StringBuilder();
-
- for (Gene gene : genes) {
- sb.append("\nName: ").append(((Movie)gene.getVal()).getName())
- .append("\nGenres: ").append(((Movie)gene.getVal()).getGenre().toString())
- .append("\nIMDB Rating: ").append(((Movie)gene.getVal()).getImdbRating());
- }
-
- return sb.toString();
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/inference/IgniteModelDistributedInferenceExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/inference/IgniteModelDistributedInferenceExample.java
index 26cf41c..deb7de2 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/inference/IgniteModelDistributedInferenceExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/inference/IgniteModelDistributedInferenceExample.java
@@ -27,6 +27,8 @@
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.examples.ml.regression.linear.LinearRegressionLSQRTrainerExample;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.inference.Model;
@@ -38,8 +40,6 @@
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.regressions.linear.LinearRegressionLSQRTrainer;
import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* This example is based on {@link LinearRegressionLSQRTrainerExample}, but to perform inference it uses an approach
@@ -101,7 +101,8 @@
System.out.println(">>> Linear regression model over cache based dataset usage example completed.");
}
finally {
- dataCache.destroy();
+ if (dataCache != null)
+ dataCache.destroy();
}
}
finally {
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/inference/TensorFlowDistributedInferenceExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/inference/TensorFlowDistributedInferenceExample.java
deleted file mode 100644
index 0d65219..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/inference/TensorFlowDistributedInferenceExample.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.inference;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.Random;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.inference.Model;
-import org.apache.ignite.ml.inference.builder.IgniteDistributedModelBuilder;
-import org.apache.ignite.ml.inference.parser.ModelParser;
-import org.apache.ignite.ml.inference.parser.TensorFlowSavedModelModelParser;
-import org.apache.ignite.ml.inference.reader.FileSystemModelReader;
-import org.apache.ignite.ml.inference.reader.ModelReader;
-import org.apache.ignite.ml.util.MnistUtils;
-import org.tensorflow.Tensor;
-
-/**
- * This example demonstrates how to: load TensorFlow model into Java, make inference in distributed environment using
- * Apache Ignite services.
- */
-public class TensorFlowDistributedInferenceExample {
- /**
- * Path to the directory with saved TensorFlow model.
- */
- private static final String MODEL_PATH = "examples/src/main/resources/models/mnist_tf_model";
-
- /**
- * Path to the MNIST images data.
- */
- private static final String MNIST_IMG_PATH = "examples/src/main/resources/datasets/t10k-images-idx3-ubyte";
-
- /**
- * Path to the MNIST labels data.
- */
- private static final String MNIST_LBL_PATH = "examples/src/main/resources/datasets/t10k-labels-idx1-ubyte";
-
- /**
- * Run example.
- */
- public static void main(String[] args) throws IOException, ExecutionException, InterruptedException {
- try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- File mdlRsrc = IgniteUtils.resolveIgnitePath(MODEL_PATH);
- if (mdlRsrc == null)
- throw new IllegalArgumentException("Resource not found [resource_path=" + MODEL_PATH + "]");
-
- ModelReader reader = new FileSystemModelReader(mdlRsrc.getPath());
-
- ModelParser<double[], Long, ?> parser = new TensorFlowSavedModelModelParser<double[], Long>("serve")
-
- .withInput("Placeholder", doubles -> {
- float[][][] reshaped = new float[1][28][28];
- for (int i = 0; i < doubles.length; i++)
- reshaped[0][i / 28][i % 28] = (float)doubles[i];
- return Tensor.create(reshaped);
- })
-
- .withOutput(Collections.singletonList("ArgMax"), collectedTensors -> collectedTensors.get("ArgMax")
- .copyTo(new long[1])[0]);
-
- List<MnistUtils.MnistLabeledImage> images = MnistUtils.mnistAsList(
- Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_IMG_PATH)).getPath(),
- Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_LBL_PATH)).getPath(),
- new Random(0),
- 10000
- );
-
- long t0 = System.currentTimeMillis();
-
- try (Model<double[], Future<Long>> threadedMdl = new IgniteDistributedModelBuilder(ignite, 4, 4)
- .build(reader, parser)) {
- List<Future<?>> futures = new ArrayList<>(images.size());
- for (MnistUtils.MnistLabeledImage image : images)
- futures.add(threadedMdl.predict(image.getPixels()));
- for (Future<?> f : futures)
- f.get();
- }
-
- long t1 = System.currentTimeMillis();
-
- System.out.println("Threaded model throughput: " + images.size() / ((t1 - t0) / 1000.0) + " req/sec");
- }
- finally {
- System.out.flush();
- }
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/inference/TensorFlowLocalInferenceExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/inference/TensorFlowLocalInferenceExample.java
deleted file mode 100644
index f77ac82..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/inference/TensorFlowLocalInferenceExample.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.inference;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.Random;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.inference.Model;
-import org.apache.ignite.ml.inference.builder.SingleModelBuilder;
-import org.apache.ignite.ml.inference.parser.ModelParser;
-import org.apache.ignite.ml.inference.parser.TensorFlowSavedModelModelParser;
-import org.apache.ignite.ml.inference.reader.FileSystemModelReader;
-import org.apache.ignite.ml.inference.reader.ModelReader;
-import org.apache.ignite.ml.util.MnistUtils;
-import org.tensorflow.Tensor;
-
-/**
- * This example demonstrates how to: load TensorFlow model into Java, make inference using this model in one thread.
- */
-public class TensorFlowLocalInferenceExample {
- /**
- * Path to the directory with saved TensorFlow model.
- */
- private static final String MODEL_PATH = "examples/src/main/resources/models/mnist_tf_model";
-
- /**
- * Path to the MNIST images data.
- */
- private static final String MNIST_IMG_PATH = "examples/src/main/resources/datasets/t10k-images-idx3-ubyte";
-
- /**
- * Path to the MNIST labels data.
- */
- private static final String MNIST_LBL_PATH = "examples/src/main/resources/datasets/t10k-labels-idx1-ubyte";
-
- /**
- * Run example.
- */
- public static void main(String[] args) throws IOException {
- File mdlRsrc = IgniteUtils.resolveIgnitePath(MODEL_PATH);
- if (mdlRsrc == null)
- throw new IllegalArgumentException("Resource not found [resource_path=" + MODEL_PATH + "]");
-
- ModelReader reader = new FileSystemModelReader(mdlRsrc.getPath());
-
- ModelParser<double[], Long, ?> parser = new TensorFlowSavedModelModelParser<double[], Long>("serve")
- .withInput("Placeholder", doubles -> {
- float[][][] reshaped = new float[1][28][28];
- for (int i = 0; i < doubles.length; i++)
- reshaped[0][i / 28][i % 28] = (float)doubles[i];
- return Tensor.create(reshaped);
- })
- .withOutput(Collections.singletonList("ArgMax"), collectedTensors -> collectedTensors.get("ArgMax")
- .copyTo(new long[1])[0]);
-
- List<MnistUtils.MnistLabeledImage> images = MnistUtils.mnistAsList(
- Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_IMG_PATH)).getPath(),
- Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_LBL_PATH)).getPath(),
- new Random(0),
- 10000
- );
-
- long t0 = System.currentTimeMillis();
-
- try (Model<double[], Long> locMdl = new SingleModelBuilder().build(reader, parser)) {
- for (MnistUtils.MnistLabeledImage image : images)
- locMdl.predict(image.getPixels());
- }
-
- long t1 = System.currentTimeMillis();
-
- System.out.println("Threaded model throughput: " + 1.0 * images.size() / ((t1 - t0) / 1000) + " req/sec");
- System.out.flush();
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/inference/TensorFlowThreadedInferenceExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/inference/TensorFlowThreadedInferenceExample.java
deleted file mode 100644
index 22f0b23..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/inference/TensorFlowThreadedInferenceExample.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.inference;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.Random;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.inference.Model;
-import org.apache.ignite.ml.inference.builder.ThreadedModelBuilder;
-import org.apache.ignite.ml.inference.parser.ModelParser;
-import org.apache.ignite.ml.inference.parser.TensorFlowSavedModelModelParser;
-import org.apache.ignite.ml.inference.reader.FileSystemModelReader;
-import org.apache.ignite.ml.inference.reader.ModelReader;
-import org.apache.ignite.ml.util.MnistUtils;
-import org.tensorflow.Tensor;
-
-/**
- * This example demonstrates how to: load TensorFlow model into Java, make inference using this model in multiple
- * threads.
- */
-public class TensorFlowThreadedInferenceExample {
- /**
- * Path to the directory with saved TensorFlow model.
- */
- private static final String MODEL_PATH = "examples/src/main/resources/models/mnist_tf_model";
-
- /**
- * Path to the MNIST images data.
- */
- private static final String MNIST_IMG_PATH = "examples/src/main/resources/datasets/t10k-images-idx3-ubyte";
-
- /**
- * Path to the MNIST labels data.
- */
- private static final String MNIST_LBL_PATH = "examples/src/main/resources/datasets/t10k-labels-idx1-ubyte";
-
- /**
- * Run example.
- */
- public static void main(String[] args) throws IOException, ExecutionException, InterruptedException {
- File mdlRsrc = IgniteUtils.resolveIgnitePath(MODEL_PATH);
- if (mdlRsrc == null)
- throw new IllegalArgumentException("Resource not found [resource_path=" + MODEL_PATH + "]");
-
- ModelReader reader = new FileSystemModelReader(mdlRsrc.getPath());
-
- ModelParser<double[], Long, ?> parser = new TensorFlowSavedModelModelParser<double[], Long>("serve")
-
- .withInput("Placeholder", doubles -> {
- float[][][] reshaped = new float[1][28][28];
- for (int i = 0; i < doubles.length; i++)
- reshaped[0][i / 28][i % 28] = (float)doubles[i];
- return Tensor.create(reshaped);
- })
-
- .withOutput(Collections.singletonList("ArgMax"), collectedTensors -> collectedTensors.get("ArgMax")
- .copyTo(new long[1])[0]);
-
- List<MnistUtils.MnistLabeledImage> images = MnistUtils.mnistAsList(
- Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_IMG_PATH)).getPath(),
- Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_LBL_PATH)).getPath(),
- new Random(0),
- 10000
- );
-
- long t0 = System.currentTimeMillis();
-
- try (Model<double[], Future<Long>> threadedMdl = new ThreadedModelBuilder(8)
- .build(reader, parser)) {
- List<Future<?>> futures = new ArrayList<>(images.size());
- for (MnistUtils.MnistLabeledImage image : images)
- futures.add(threadedMdl.predict(image.getPixels()));
- for (Future<?> f : futures)
- f.get();
- }
-
- long t1 = System.currentTimeMillis();
-
- System.out.println("Threaded model throughput: " + 1.0 * images.size() / ((t1 - t0) / 1000) + " req/sec");
- System.out.flush();
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/inference/spark/LogRegFromSparkThroughPMMLExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/inference/spark/LogRegFromSparkThroughPMMLExample.java
index 7fa112c..f1beb70 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/inference/spark/LogRegFromSparkThroughPMMLExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/inference/spark/LogRegFromSparkThroughPMMLExample.java
@@ -25,6 +25,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
@@ -33,8 +35,6 @@
import org.apache.ignite.ml.regressions.logistic.LogisticRegressionModel;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.classification.Accuracy;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
import org.dmg.pmml.PMML;
import org.dmg.pmml.regression.RegressionModel;
import org.dmg.pmml.regression.RegressionTable;
@@ -81,8 +81,9 @@
System.out.println("\n>>> Accuracy " + accuracy);
System.out.println("\n>>> Test Error " + (1 - accuracy));
- } finally {
- dataCache.destroy();
+ } finally {
+ if (dataCache != null)
+ dataCache.destroy();
}
}
}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/xgboost/XGBoostModelParserExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/inference/xgboost/XGBoostModelParserExample.java
similarity index 98%
rename from examples/src/main/java/org/apache/ignite/examples/ml/xgboost/XGBoostModelParserExample.java
rename to examples/src/main/java/org/apache/ignite/examples/ml/inference/xgboost/XGBoostModelParserExample.java
index 9f9c6f9..997ece8 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/xgboost/XGBoostModelParserExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/inference/xgboost/XGBoostModelParserExample.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.ignite.examples.ml.xgboost;
+package org.apache.ignite.examples.ml.inference.xgboost;
import java.io.File;
import java.io.FileNotFoundException;
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/xgboost/package-info.java b/examples/src/main/java/org/apache/ignite/examples/ml/inference/xgboost/package-info.java
similarity index 93%
rename from examples/src/main/java/org/apache/ignite/examples/ml/xgboost/package-info.java
rename to examples/src/main/java/org/apache/ignite/examples/ml/inference/xgboost/package-info.java
index f0c8db4..3f834d2 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/xgboost/package-info.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/inference/xgboost/package-info.java
@@ -19,4 +19,4 @@
* <!-- Package description. --> XGBoost model inference examples.
*/
-package org.apache.ignite.examples.ml.xgboost;
+package org.apache.ignite.examples.ml.inference.xgboost;
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/knn/IrisClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/knn/IrisClassificationExample.java
index c06c5c3..e8ef55e 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/knn/IrisClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/knn/IrisClassificationExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
@@ -32,8 +34,6 @@
import org.apache.ignite.ml.selection.scoring.metric.classification.Accuracy;
import org.apache.ignite.ml.selection.split.TrainTestDatasetSplitter;
import org.apache.ignite.ml.selection.split.TrainTestSplit;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Example of using Knn model in Apache Ignite for iris class predicion.
@@ -87,7 +87,8 @@
System.out.println(">> Model accuracy: " + accuracy);
}
finally {
- dataCache.destroy();
+ if (dataCache != null)
+ dataCache.destroy();
}
}
finally {
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
index e089f51..1917d22 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.knn.classification.KNNClassificationModel;
@@ -29,8 +31,6 @@
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.MetricName;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Run kNN multi-class classification trainer ({@link KNNClassificationTrainer}) over distributed dataset.
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNRegressionExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNRegressionExample.java
index 80d8a1a..a2c1cbe 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNRegressionExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNRegressionExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.knn.regression.KNNRegressionModel;
@@ -30,8 +32,6 @@
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.regression.Rmse;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Run kNN regression trainer ({@link KNNRegressionTrainer}) over distributed dataset.
@@ -84,7 +84,8 @@
System.out.println("\n>>> Rmse = " + rmse);
}
finally {
- dataCache.destroy();
+ if (dataCache != null)
+ dataCache.destroy();
}
}
finally {
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/multiclass/OneVsRestClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/multiclass/OneVsRestClassificationExample.java
index 7201899..dd5fecf 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/multiclass/OneVsRestClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/multiclass/OneVsRestClassificationExample.java
@@ -26,6 +26,8 @@
import org.apache.ignite.Ignition;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.multiclass.MultiClassModel;
@@ -34,8 +36,6 @@
import org.apache.ignite.ml.preprocessing.minmaxscaling.MinMaxScalerTrainer;
import org.apache.ignite.ml.svm.SVMLinearClassificationModel;
import org.apache.ignite.ml.svm.SVMLinearClassificationTrainer;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Run One-vs-Rest multi-class classification trainer ({@link OneVsRestTrainer}) parametrized by binary SVM classifier
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/naivebayes/CompoundNaiveBayesExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/naivebayes/CompoundNaiveBayesExample.java
index 0f7e3a1..24bbcb8 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/naivebayes/CompoundNaiveBayesExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/naivebayes/CompoundNaiveBayesExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -30,8 +32,6 @@
import org.apache.ignite.ml.naivebayes.gaussian.GaussianNaiveBayesTrainer;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.MetricName;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
import static java.util.Arrays.asList;
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/naivebayes/DiscreteNaiveBayesTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/naivebayes/DiscreteNaiveBayesTrainerExample.java
index c62c53a..2aee5fe 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/naivebayes/DiscreteNaiveBayesTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/naivebayes/DiscreteNaiveBayesTrainerExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -28,8 +30,6 @@
import org.apache.ignite.ml.naivebayes.discrete.DiscreteNaiveBayesTrainer;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.MetricName;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Run naive Bayes classification model based on <a href=https://en.wikipedia.org/wiki/Naive_Bayes_classifier#Multinomial_naive_Bayes">
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/naivebayes/GaussianNaiveBayesTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/naivebayes/GaussianNaiveBayesTrainerExample.java
index 15ac6cb..6571d3e 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/naivebayes/GaussianNaiveBayesTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/naivebayes/GaussianNaiveBayesTrainerExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -28,8 +30,6 @@
import org.apache.ignite.ml.naivebayes.gaussian.GaussianNaiveBayesTrainer;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.MetricName;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Run naive Bayes classification model based on <a href="https://en.wikipedia.org/wiki/Naive_Bayes_classifier"> naive
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/encoding/EncoderExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/encoding/EncoderExample.java
index c189c1c..c24091c 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/encoding/EncoderExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/encoding/EncoderExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.ObjectArrayVectorizer;
import org.apache.ignite.ml.preprocessing.Preprocessor;
@@ -30,8 +32,6 @@
import org.apache.ignite.ml.selection.scoring.metric.classification.Accuracy;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Example that shows how to use String Encoder preprocessor to encode features presented as a strings.
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/encoding/EncoderExampleWithNormalization.java b/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/encoding/EncoderExampleWithNormalization.java
index 3af14e3..d9482a5 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/encoding/EncoderExampleWithNormalization.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/encoding/EncoderExampleWithNormalization.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.ObjectArrayVectorizer;
import org.apache.ignite.ml.preprocessing.Preprocessor;
@@ -31,8 +33,6 @@
import org.apache.ignite.ml.selection.scoring.metric.classification.Accuracy;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Example that shows how to combine together two preprocessors: String Encoder preprocessor to encode features presented as a strings
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/encoding/LabelEncoderExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/encoding/LabelEncoderExample.java
index 675336f..d97c49c 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/encoding/LabelEncoderExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/encoding/LabelEncoderExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.ObjectArrayVectorizer;
import org.apache.ignite.ml.preprocessing.Preprocessor;
@@ -30,8 +32,6 @@
import org.apache.ignite.ml.selection.scoring.metric.classification.Accuracy;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Example that shows how to use Label Encoder preprocessor to encode labels presented as a strings.
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/recommendation/MovieLensExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/recommendation/MovieLensExample.java
index 35e8151..b5c7984 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/recommendation/MovieLensExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/recommendation/MovieLensExample.java
@@ -26,13 +26,13 @@
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
import org.apache.ignite.ml.recommendation.ObjectSubjectRatingTriplet;
import org.apache.ignite.ml.recommendation.RecommendationModel;
import org.apache.ignite.ml.recommendation.RecommendationTrainer;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Example of recommendation system based on MovieLens dataset (see https://grouplens.org/datasets/movielens/).
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/recommendation/MovieLensSQLExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/recommendation/MovieLensSQLExample.java
index b316c61..dbfb553 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/recommendation/MovieLensSQLExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/recommendation/MovieLensSQLExample.java
@@ -26,14 +26,14 @@
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
import org.apache.ignite.ml.inference.IgniteModelStorageUtil;
import org.apache.ignite.ml.recommendation.RecommendationModel;
import org.apache.ignite.ml.recommendation.RecommendationTrainer;
import org.apache.ignite.ml.sql.SQLFunctions;
import org.apache.ignite.ml.sql.SqlDatasetBuilder;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Example of recommendation system based on MovieLens dataset (see https://grouplens.org/datasets/movielens/) and SQL.
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/BostonHousePricesPredictionExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/BostonHousePricesPredictionExample.java
index afaa4c9..511eb05 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/BostonHousePricesPredictionExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/BostonHousePricesPredictionExample.java
@@ -22,6 +22,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
@@ -33,8 +35,6 @@
import org.apache.ignite.ml.selection.split.TrainTestDatasetSplitter;
import org.apache.ignite.ml.selection.split.TrainTestSplit;
import org.apache.ignite.ml.trainers.DatasetTrainer;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Example of using Linear Regression model in Apache Ignite for house prices prediction.
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerExample.java
index c4da76c..e6ab6f6e 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -28,8 +30,6 @@
import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.MetricName;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Run linear regression model based on <a href="http://web.stanford.edu/group/SOL/software/lsqr/">LSQR algorithm</a>
@@ -87,7 +87,8 @@
System.out.println(">>> Linear regression model over cache based dataset usage example completed.");
}
finally {
- dataCache.destroy();
+ if (dataCache != null)
+ dataCache.destroy();
}
}
finally {
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerWithMinMaxScalerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerWithMinMaxScalerExample.java
index 94c94b1..9979d3c 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerWithMinMaxScalerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerWithMinMaxScalerExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -31,8 +33,6 @@
import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.MetricName;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Run linear regression model based on <a href="http://web.stanford.edu/group/SOL/software/lsqr/">LSQR algorithm</a>
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionSGDTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionSGDTrainerExample.java
index 2b59d6a..5f116c4 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionSGDTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionSGDTrainerExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -31,8 +33,6 @@
import org.apache.ignite.ml.regressions.linear.LinearRegressionSGDTrainer;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.MetricName;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Run linear regression model based on based on
@@ -88,7 +88,8 @@
System.out.println(">>> Linear regression model over cache based dataset usage example completed.");
}
finally {
- dataCache.destroy();
+ if (dataCache != null)
+ dataCache.destroy();
}
}
finally {
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/bagged/BaggedLogisticRegressionSGDTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/bagged/BaggedLogisticRegressionSGDTrainerExample.java
index daa0c15..d7f63d2 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/bagged/BaggedLogisticRegressionSGDTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/bagged/BaggedLogisticRegressionSGDTrainerExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.composition.bagging.BaggedTrainer;
import org.apache.ignite.ml.composition.predictionsaggregator.OnMajorityPredictionsAggregator;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
@@ -34,8 +36,6 @@
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.MetricName;
import org.apache.ignite.ml.trainers.TrainerTransformers;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* This example shows how bagging technique may be applied to arbitrary trainer. As an example (a bit synthetic)
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java
index f6a4b09..fd6b9ff 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -31,8 +33,6 @@
import org.apache.ignite.ml.regressions.logistic.LogisticRegressionSGDTrainer;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.MetricName;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Run logistic regression model based on <a href="https://en.wikipedia.org/wiki/Stochastic_gradient_descent">
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/selection/scoring/EvaluatorExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/selection/scoring/EvaluatorExample.java
index a53f16a..89e71fa 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/selection/scoring/EvaluatorExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/selection/scoring/EvaluatorExample.java
@@ -21,14 +21,14 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.svm.SVMLinearClassificationModel;
import org.apache.ignite.ml.svm.SVMLinearClassificationTrainer;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Run SVM classification trainer ({@link SVMLinearClassificationTrainer}) over distributed dataset.
@@ -69,7 +69,8 @@
System.out.println(Evaluator.evaluateBinaryClassification(dataCache, mdl, vectorizer));
}
finally {
- dataCache.destroy();
+ if (dataCache != null)
+ dataCache.destroy();
}
}
finally {
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/selection/scoring/MultipleMetricsExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/selection/scoring/MultipleMetricsExample.java
deleted file mode 100644
index 7d24afd..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/selection/scoring/MultipleMetricsExample.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.selection.scoring;
-
-import java.io.IOException;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
-import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
-import org.apache.ignite.ml.knn.classification.KNNClassificationTrainer;
-import org.apache.ignite.ml.math.primitives.vector.Vector;
-import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
-import org.apache.ignite.ml.svm.SVMLinearClassificationModel;
-import org.apache.ignite.ml.svm.SVMLinearClassificationTrainer;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
-
-/**
- * Run kNN multi-class classification trainer ({@link KNNClassificationTrainer}) over distributed dataset.
- * <p>
- * Code in this example launches Ignite grid and fills the cache with test data points (based on the
- * <a href="https://en.wikipedia.org/wiki/Iris_flower_data_set"></a>Iris dataset</a>).</p>
- * <p>
- * After that it trains the model based on the specified data using
- * <a href="https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm">kNN</a> algorithm.</p>
- * <p>
- * Finally, this example loops over the test set of data points, applies the trained model to predict what cluster does
- * this point belong to, and compares prediction to expected outcome (ground truth).</p>
- * <p>
- * You can change the test data used in this example and re-run it to explore this algorithm further.</p>
- */
-public class MultipleMetricsExample {
- /**
- * Run example.
- */
- public static void main(String[] args) throws IOException {
- // Start ignite grid.
- try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- System.out.println(">>> Ignite grid started.");
-
- IgniteCache<Integer, Vector> dataCache = null;
- try {
- dataCache = new SandboxMLCache(ignite).fillCacheWith(MLSandboxDatasets.TWO_CLASSED_IRIS);
-
- SVMLinearClassificationTrainer trainer = new SVMLinearClassificationTrainer();
-
- Vectorizer<Integer, Vector, Integer, Double> vectorizer = new DummyVectorizer<Integer>()
- .labeled(Vectorizer.LabelCoordinate.FIRST);
-
- SVMLinearClassificationModel mdl = trainer.fit(ignite, dataCache, vectorizer);
-
- System.out.println(Evaluator.evaluateBinaryClassification(dataCache, mdl, vectorizer));
- }
- finally {
- dataCache.destroy();
- }
- }
- finally {
- System.out.flush();
- }
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/selection/scoring/RegressionMetricExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/selection/scoring/RegressionMetricExample.java
index 448960d..67b977f 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/selection/scoring/RegressionMetricExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/selection/scoring/RegressionMetricExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.knn.regression.KNNRegressionModel;
@@ -30,8 +32,6 @@
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.MetricName;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Run kNN regression trainer ({@link KNNRegressionTrainer}) over distributed dataset.
@@ -78,7 +78,8 @@
System.out.println("\n>>> Mae " + mae);
}
finally {
- dataCache.destroy();
+ if (dataCache != null)
+ dataCache.destroy();
}
}
finally {
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/selection/split/TrainTestDatasetSplitterExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/selection/split/TrainTestDatasetSplitterExample.java
index 04fe5e0..5c7f02ed 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/selection/split/TrainTestDatasetSplitterExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/selection/split/TrainTestDatasetSplitterExample.java
@@ -24,6 +24,8 @@
import org.apache.ignite.Ignition;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -31,8 +33,6 @@
import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
import org.apache.ignite.ml.selection.split.TrainTestDatasetSplitter;
import org.apache.ignite.ml.selection.split.TrainTestSplit;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Run linear regression model over dataset split on train and test subsets ({@link TrainTestDatasetSplitter}).
@@ -100,7 +100,8 @@
System.out.println(">>> Linear regression model over cache based dataset usage example completed.");
}
finally {
- dataCache.destroy();
+ if (dataCache != null)
+ dataCache.destroy();
}
}
finally {
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMBinaryClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMBinaryClassificationExample.java
index 3850749..20b3b2f 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMBinaryClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMBinaryClassificationExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -28,8 +30,6 @@
import org.apache.ignite.ml.selection.scoring.metric.MetricName;
import org.apache.ignite.ml.svm.SVMLinearClassificationModel;
import org.apache.ignite.ml.svm.SVMLinearClassificationTrainer;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Run SVM binary-class classification model ({@link SVMLinearClassificationModel}) over distributed dataset.
@@ -78,7 +78,8 @@
System.out.println(">>> SVM Binary classification model over cache based dataset usage example completed.");
}
finally {
- dataCache.destroy();
+ if (dataCache != null)
+ dataCache.destroy();
}
}
finally {
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/FraudDetectionExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/FraudDetectionExample.java
index 1dab807..ed758d4 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/FraudDetectionExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/FraudDetectionExample.java
@@ -21,6 +21,8 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.IgniteModel;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
@@ -32,8 +34,6 @@
import org.apache.ignite.ml.selection.split.TrainTestSplit;
import org.apache.ignite.ml.trainers.DatasetTrainer;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Example of using classification algorithms for fraud detection problem.
@@ -79,7 +79,8 @@
);
}
finally {
- dataCache.destroy();
+ if (dataCache != null)
+ dataCache.destroy();
}
}
finally {
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestClassificationExample.java
index 15cc5d4..0d9dbef 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestClassificationExample.java
@@ -28,6 +28,8 @@
import org.apache.ignite.Ignition;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.composition.ModelsComposition;
import org.apache.ignite.ml.dataset.feature.FeatureMeta;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
@@ -35,8 +37,6 @@
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.tree.randomforest.RandomForestClassifierTrainer;
import org.apache.ignite.ml.tree.randomforest.data.FeaturesCountSelectionStrategies;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Example represents a solution for the task of wine classification based on a
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestRegressionExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestRegressionExample.java
index cb2b1d0..714589a 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestRegressionExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestRegressionExample.java
@@ -27,6 +27,8 @@
import org.apache.ignite.Ignition;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.examples.ml.util.MLSandboxDatasets;
+import org.apache.ignite.examples.ml.util.SandboxMLCache;
import org.apache.ignite.ml.composition.ModelsComposition;
import org.apache.ignite.ml.dataset.feature.FeatureMeta;
import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
@@ -37,8 +39,6 @@
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.tree.randomforest.RandomForestRegressionTrainer;
import org.apache.ignite.ml.tree.randomforest.data.FeaturesCountSelectionStrategies;
-import org.apache.ignite.ml.util.MLSandboxDatasets;
-import org.apache.ignite.ml.util.SandboxMLCache;
/**
* Example represents a solution for the task of price predictions for houses in Boston based on a
@@ -125,7 +125,8 @@
}
}
finally {
- dataCache.destroy();
+ if (dataCache != null)
+ dataCache.destroy();
}
}
finally {
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid.java
index c2b2c1a..5b62714 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid.java
@@ -123,7 +123,7 @@
= new CrossValidation<>();
ParamGrid paramGrid = new ParamGrid()
- .addHyperParam("maxDeep", trainerCV::withMaxDeep, new Double[] {1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 10.0})
+ .addHyperParam("maxDeep", trainerCV::withMaxDeep, new Double[] {1.0, 2.0, 3.0, 4.0, 5.0, 10.0})
.addHyperParam("minImpurityDecrease", trainerCV::withMinImpurityDecrease, new Double[] {0.0, 0.25, 0.5});
scoreCalculator
@@ -175,7 +175,7 @@
System.out.println("\n>>> Accuracy " + accuracy);
System.out.println("\n>>> Test Error " + (1 - accuracy));
- System.out.println(">>> Tutorial step 8 (cross-validation with param grid) example started.");
+ System.out.println(">>> Tutorial step 8 (cross-validation with param grid) example completed.");
}
catch (FileNotFoundException e) {
e.printStackTrace();
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid_and_metrics.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid_and_metrics.java
deleted file mode 100644
index ef1defd..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid_and_metrics.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.tutorial;
-
-import java.io.FileNotFoundException;
-import java.util.Arrays;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
-import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
-import org.apache.ignite.ml.math.primitives.vector.Vector;
-import org.apache.ignite.ml.preprocessing.Preprocessor;
-import org.apache.ignite.ml.preprocessing.encoding.EncoderTrainer;
-import org.apache.ignite.ml.preprocessing.encoding.EncoderType;
-import org.apache.ignite.ml.preprocessing.imputing.ImputerTrainer;
-import org.apache.ignite.ml.preprocessing.minmaxscaling.MinMaxScalerTrainer;
-import org.apache.ignite.ml.preprocessing.normalization.NormalizationTrainer;
-import org.apache.ignite.ml.selection.cv.CrossValidation;
-import org.apache.ignite.ml.selection.cv.CrossValidationResult;
-import org.apache.ignite.ml.selection.paramgrid.ParamGrid;
-import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
-import org.apache.ignite.ml.selection.scoring.metric.MetricName;
-import org.apache.ignite.ml.selection.scoring.metric.classification.Accuracy;
-import org.apache.ignite.ml.selection.split.TrainTestDatasetSplitter;
-import org.apache.ignite.ml.selection.split.TrainTestSplit;
-import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
-import org.apache.ignite.ml.tree.DecisionTreeNode;
-
-/**
- * To choose the best hyper-parameters the cross-validation with {@link ParamGrid} will be used in this example.
- * <p>
- * Code in this example launches Ignite grid and fills the cache with test data (based on Titanic passengers data).</p>
- * <p>
- * After that it defines how to split the data to train and test sets and configures preprocessors that extract features
- * from an upstream data and perform other desired changes over the extracted data.</p>
- * <p>
- * Then, it tunes hyper-parameters with K-fold Cross-Validation on the split training set and trains the model based on the
- * processed data using decision tree classification and the obtained hyper-parameters.</p>
- * <p>
- * Finally, this example uses {@link Evaluator} functionality to compute metrics from predictions.</p>
- * <p>
- * The purpose of cross-validation is model checking, not model building.</p>
- * <p>
- * We train {@code k} different models.</p>
- * <p>
- * They differ in that {@code 1/(k-1)}th of the training data is exchanged against other cases.</p>
- * <p>
- * These models are sometimes called surrogate models because the (average) performance measured for these models is
- * taken as a surrogate of the performance of the model trained on all cases.</p>
- * <p>
- * All scenarios are described there: https://sebastianraschka.com/faq/docs/evaluate-a-model.html</p>
- */
-public class Step_8_CV_with_Param_Grid_and_metrics {
- /**
- * Run example.
- */
- public static void main(String[] args) {
- System.out.println();
- System.out.println(">>> Tutorial step 8 (cross-validation with param grid) example started.");
-
- try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- try {
- IgniteCache<Integer, Vector> dataCache = TitanicUtils.readPassengers(ignite);
-
- // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare".
- final Vectorizer<Integer, Vector, Integer, Double> vectorizer
- = new DummyVectorizer<Integer>(0, 3, 4, 5, 6, 8, 10).labeled(1);
-
- TrainTestSplit<Integer, Vector> split = new TrainTestDatasetSplitter<Integer, Vector>()
- .split(0.75);
-
- Preprocessor<Integer, Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Vector>()
- .withEncoderType(EncoderType.STRING_ENCODER)
- .withEncodedFeature(1)
- .withEncodedFeature(6) // <--- Changed index here.
- .fit(ignite,
- dataCache,
- vectorizer
- );
-
- Preprocessor<Integer, Vector> imputingPreprocessor = new ImputerTrainer<Integer, Vector>()
- .fit(ignite,
- dataCache,
- strEncoderPreprocessor
- );
-
- Preprocessor<Integer, Vector> minMaxScalerPreprocessor = new MinMaxScalerTrainer<Integer, Vector>()
- .fit(
- ignite,
- dataCache,
- imputingPreprocessor
- );
-
- Preprocessor<Integer, Vector> normalizationPreprocessor = new NormalizationTrainer<Integer, Vector>()
- .withP(1)
- .fit(
- ignite,
- dataCache,
- minMaxScalerPreprocessor
- );
-
- // Tune hyper-parameters with K-fold Cross-Validation on the split training set.
-
- DecisionTreeClassificationTrainer trainerCV = new DecisionTreeClassificationTrainer();
-
- CrossValidation<DecisionTreeNode, Integer, Vector> scoreCalculator
- = new CrossValidation<>();
-
- ParamGrid paramGrid = new ParamGrid()
- .addHyperParam("maxDeep", trainerCV::withMaxDeep, new Double[] {1.0, 2.0, 3.0, 4.0, 5.0, 10.0})
- .addHyperParam("minImpurityDecrease", trainerCV::withMinImpurityDecrease, new Double[] {0.0, 0.25, 0.5});
-
- scoreCalculator
- .withIgnite(ignite)
- .withUpstreamCache(dataCache)
- .withTrainer(trainerCV)
- .withMetric(MetricName.ACCURACY)
- .withFilter(split.getTrainFilter())
- .isRunningOnPipeline(false)
- .withPreprocessor(normalizationPreprocessor)
- .withAmountOfFolds(3)
- .withParamGrid(paramGrid);
-
- CrossValidationResult crossValidationRes = scoreCalculator.tuneHyperParameters();
-
- System.out.println("Train with maxDeep: " + crossValidationRes.getBest("maxDeep")
- + " and minImpurityDecrease: " + crossValidationRes.getBest("minImpurityDecrease"));
-
- DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer()
- .withMaxDeep(crossValidationRes.getBest("maxDeep"))
- .withMinImpurityDecrease(crossValidationRes.getBest("minImpurityDecrease"));
-
- System.out.println(crossValidationRes);
-
- System.out.println("Best score: " + Arrays.toString(crossValidationRes.getBestScore()));
- System.out.println("Best hyper params: " + crossValidationRes.getBestHyperParams());
- System.out.println("Best average score: " + crossValidationRes.getBestAvgScore());
-
- crossValidationRes.getScoringBoard().forEach((hyperParams, score)
- -> System.out.println("Score " + Arrays.toString(score) + " for hyper params " + hyperParams));
-
- // Train decision tree model.
- DecisionTreeNode bestMdl = trainer.fit(
- ignite,
- dataCache,
- split.getTrainFilter(),
- normalizationPreprocessor
- );
-
- System.out.println("\n>>> Trained model: " + bestMdl);
-
- double accuracy = Evaluator.evaluate(
- dataCache, split.getTestFilter(),
- bestMdl,
- normalizationPreprocessor,
- new Accuracy<>()
- );
-
- System.out.println("\n>>> Accuracy " + accuracy);
- System.out.println("\n>>> Test Error " + (1 - accuracy));
-
- System.out.println(">>> Tutorial step 8 (cross-validation with param grid) example started.");
- }
- catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- }
- finally {
- System.out.flush();
- }
- }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid_and_metrics_and_pipeline.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid_and_pipeline.java
similarity index 96%
rename from examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid_and_metrics_and_pipeline.java
rename to examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid_and_pipeline.java
index 24f2d6d..6be8496 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid_and_metrics_and_pipeline.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid_and_pipeline.java
@@ -62,13 +62,13 @@
* <p>
* All scenarios are described there: https://sebastianraschka.com/faq/docs/evaluate-a-model.html</p>
*/
-public class Step_8_CV_with_Param_Grid_and_metrics_and_pipeline {
+public class Step_8_CV_with_Param_Grid_and_pipeline {
/**
* Run example.
*/
public static void main(String[] args) {
System.out.println();
- System.out.println(">>> Tutorial step 8 (cross-validation with param grid) example started.");
+ System.out.println(">>> Tutorial step 8 (cross-validation with param grid and pipeline) example started.");
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
try {
@@ -104,7 +104,6 @@
.withPipeline(pipeline)
.withMetric(MetricName.ACCURACY)
.withFilter(split.getTrainFilter())
- .withPreprocessor(vectorizer)
.withAmountOfFolds(3)
.withParamGrid(paramGrid);
@@ -122,6 +121,7 @@
crossValidationRes.getScoringBoard().forEach((hyperParams, score)
-> System.out.println("Score " + Arrays.toString(score) + " for hyper params " + hyperParams));
+ System.out.println(">>> Tutorial step 8 (cross-validation with param grid and pipeline) example completed.");
}
catch (FileNotFoundException e) {
e.printStackTrace();
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_9_Scaling_With_Stacking.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_9_Scaling_With_Stacking.java
index bb710d3..7199257 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_9_Scaling_With_Stacking.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_9_Scaling_With_Stacking.java
@@ -138,7 +138,7 @@
System.out.println("\n>>> Accuracy " + accuracy);
System.out.println("\n>>> Test Error " + (1 - accuracy));
- System.out.println(">>> Tutorial step 5 (scaling) example completed.");
+ System.out.println(">>> Tutorial step 9 (scaling with stacking) example completed.");
}
catch (FileNotFoundException e) {
e.printStackTrace();
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java
index 5dd1551..8ef5fe0 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java
@@ -18,7 +18,7 @@
package org.apache.ignite.examples.ml.tutorial;
import org.apache.ignite.examples.ml.tutorial.hyperparametertuning.Step_13_RandomSearch;
-import org.apache.ignite.examples.ml.tutorial.hyperparametertuning.Step_14_Parallel_BrutForce_Search;
+import org.apache.ignite.examples.ml.tutorial.hyperparametertuning.Step_14_Parallel_Brute_Force_Search;
import org.apache.ignite.examples.ml.tutorial.hyperparametertuning.Step_15_Parallel_Random_Search;
import org.apache.ignite.examples.ml.tutorial.hyperparametertuning.Step_16_Genetic_Programming_Search;
import org.apache.ignite.examples.ml.tutorial.hyperparametertuning.Step_17_Parallel_Genetic_Programming_Search;
@@ -42,12 +42,13 @@
Step_7_Split_train_test.main(args);
Step_8_CV.main(args);
Step_8_CV_with_Param_Grid.main(args);
+ Step_8_CV_with_Param_Grid_and_pipeline.main(args);
Step_9_Scaling_With_Stacking.main(args);
Step_10_Bagging.main(args);
Step_11_Boosting.main(args);
Step_12_Model_Update.main(args);
Step_13_RandomSearch.main(args);
- Step_14_Parallel_BrutForce_Search.main(args);
+ Step_14_Parallel_Brute_Force_Search.main(args);
Step_15_Parallel_Random_Search.main(args);
Step_16_Genetic_Programming_Search.main(args);
Step_17_Parallel_Genetic_Programming_Search.main(args);
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_13_RandomSearch.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_13_RandomSearch.java
index dcf94a0..d7e2f27 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_13_RandomSearch.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_13_RandomSearch.java
@@ -73,6 +73,9 @@
* Run example.
*/
public static void main(String[] args) {
+ System.out.println();
+ System.out.println(">>> Tutorial step 13 (Random Search) example started.");
+
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
try {
IgniteCache<Integer, Vector> dataCache = TitanicUtils.readPassengers(ignite);
@@ -182,7 +185,7 @@
System.out.println("\n>>> Accuracy " + accuracy);
System.out.println("\n>>> Test Error " + (1 - accuracy));
- System.out.println(">>> Tutorial step 8 (cross-validation with param grid) example started.");
+ System.out.println(">>> Tutorial step 13 (Random Search) example completed.");
}
catch (FileNotFoundException e) {
e.printStackTrace();
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_14_Parallel_BrutForce_Search.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_14_Parallel_Brute_Force_Search.java
similarity index 97%
rename from examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_14_Parallel_BrutForce_Search.java
rename to examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_14_Parallel_Brute_Force_Search.java
index 2fa6b40..017f123 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_14_Parallel_BrutForce_Search.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_14_Parallel_Brute_Force_Search.java
@@ -71,11 +71,14 @@
* <p>
* All scenarios are described there: https://sebastianraschka.com/faq/docs/evaluate-a-model.html</p>
*/
-public class Step_14_Parallel_BrutForce_Search {
+public class Step_14_Parallel_Brute_Force_Search {
/**
* Run example.
*/
public static void main(String[] args) {
+ System.out.println();
+ System.out.println(">>> Tutorial step 14 (Brute Force) example started.");
+
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
try {
IgniteCache<Integer, Vector> dataCache = TitanicUtils.readPassengers(ignite);
@@ -184,7 +187,7 @@
System.out.println("\n>>> Accuracy " + accuracy);
System.out.println("\n>>> Test Error " + (1 - accuracy));
- System.out.println(">>> Tutorial step 8 (cross-validation with param grid) example started.");
+ System.out.println(">>> Tutorial step 14 (Brute Force) example completed.");
}
catch (FileNotFoundException e) {
e.printStackTrace();
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_15_Parallel_Random_Search.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_15_Parallel_Random_Search.java
index c0cf6c4..3a3e9e8 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_15_Parallel_Random_Search.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_15_Parallel_Random_Search.java
@@ -76,6 +76,9 @@
* Run example.
*/
public static void main(String[] args) {
+ System.out.println();
+ System.out.println(">>> Tutorial step 15 (Parallel Random Search) example started.");
+
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
try {
IgniteCache<Integer, Vector> dataCache = TitanicUtils.readPassengers(ignite);
@@ -187,7 +190,7 @@
System.out.println("\n>>> Accuracy " + accuracy);
System.out.println("\n>>> Test Error " + (1 - accuracy));
- System.out.println(">>> Tutorial step 8 (cross-validation with param grid) example started.");
+ System.out.println(">>> Tutorial step 15 (Parallel Random Search) example completed.");
}
catch (FileNotFoundException e) {
e.printStackTrace();
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_16_Genetic_Programming_Search.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_16_Genetic_Programming_Search.java
index 7261c5a..bee51e4 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_16_Genetic_Programming_Search.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_16_Genetic_Programming_Search.java
@@ -73,6 +73,9 @@
* Run example.
*/
public static void main(String[] args) {
+ System.out.println();
+ System.out.println(">>> Tutorial step 16 (Genetic Programming) example started.");
+
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
try {
IgniteCache<Integer, Vector> dataCache = TitanicUtils.readPassengers(ignite);
@@ -178,7 +181,7 @@
System.out.println("\n>>> Accuracy " + accuracy);
System.out.println("\n>>> Test Error " + (1 - accuracy));
- System.out.println(">>> Tutorial step 8 (cross-validation with param grid) example started.");
+ System.out.println(">>> Tutorial step 16 (Genetic Programming) example completed.");
}
catch (FileNotFoundException e) {
e.printStackTrace();
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_17_Parallel_Genetic_Programming_Search.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_17_Parallel_Genetic_Programming_Search.java
index 446d914..34a8158 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_17_Parallel_Genetic_Programming_Search.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/hyperparametertuning/Step_17_Parallel_Genetic_Programming_Search.java
@@ -76,6 +76,9 @@
* Run example.
*/
public static void main(String[] args) {
+ System.out.println();
+ System.out.println(">>> Tutorial step 17 (Parallel Genetic Programming) example started.");
+
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
try {
IgniteCache<Integer, Vector> dataCache = TitanicUtils.readPassengers(ignite);
@@ -184,7 +187,7 @@
System.out.println("\n>>> Accuracy " + accuracy);
System.out.println("\n>>> Test Error " + (1 - accuracy));
- System.out.println(">>> Tutorial step 8 (cross-validation with param grid) example started.");
+ System.out.println(">>> Tutorial step 17 (Parallel Genetic Programming) example completed.");
}
catch (FileNotFoundException e) {
e.printStackTrace();
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/util/MLSandboxDatasets.java b/examples/src/main/java/org/apache/ignite/examples/ml/util/MLSandboxDatasets.java
similarity index 71%
rename from modules/ml/src/main/java/org/apache/ignite/ml/util/MLSandboxDatasets.java
rename to examples/src/main/java/org/apache/ignite/examples/ml/util/MLSandboxDatasets.java
index 4b7a2e8..7021e7d 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/util/MLSandboxDatasets.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/util/MLSandboxDatasets.java
@@ -15,60 +15,60 @@
* limitations under the License.
*/
-package org.apache.ignite.ml.util;
+package org.apache.ignite.examples.ml.util;
/**
* The names of popular datasets used in examples.
*/
public enum MLSandboxDatasets {
/** Movielens dataset with ratings. */
- MOVIELENS("modules/ml/src/main/resources/datasets/ratings.csv", true, ","),
+ MOVIELENS("examples/src/main/resources/datasets/ratings.csv", true, ","),
/** The full Iris dataset from Machine Learning Repository. */
- IRIS("modules/ml/src/main/resources/datasets/iris.txt", false, "\t"),
+ IRIS("examples/src/main/resources/datasets/iris.txt", false, "\t"),
/** The Titanic dataset from Kaggle competition. */
- TITANIC("modules/ml/src/main/resources/datasets/titanic.csv", true, ";"),
+ TITANIC("examples/src/main/resources/datasets/titanic.csv", true, ";"),
/** The 1st and 2nd classes from the Iris dataset. */
- TWO_CLASSED_IRIS("modules/ml/src/main/resources/datasets/two_classed_iris.csv", false, "\t"),
+ TWO_CLASSED_IRIS("examples/src/main/resources/datasets/two_classed_iris.csv", false, "\t"),
/** The dataset is about different computers' properties based on https://archive.ics.uci.edu/ml/datasets/Computer+Hardware. */
- CLEARED_MACHINES("modules/ml/src/main/resources/datasets/cleared_machines.csv", false, ";"),
+ CLEARED_MACHINES("examples/src/main/resources/datasets/cleared_machines.csv", false, ";"),
/**
* The health data is related to death rate based on; doctor availability, hospital availability,
* annual per capita income, and population density people per square mile.
*/
- MORTALITY_DATA("modules/ml/src/main/resources/datasets/mortalitydata.csv", false, ";"),
+ MORTALITY_DATA("examples/src/main/resources/datasets/mortalitydata.csv", false, ";"),
/**
* The preprocessed Glass dataset from the Machine Learning Repository https://archive.ics.uci.edu/ml/datasets/Glass+Identification
* There are 3 classes with labels: 1 {building_windows_float_processed}, 3 {vehicle_windows_float_processed}, 7 {headlamps}.
* Feature names: 'Na-Sodium', 'Mg-Magnesium', 'Al-Aluminum', 'Ba-Barium', 'Fe-Iron'.
*/
- GLASS_IDENTIFICATION("modules/ml/src/main/resources/datasets/glass_identification.csv", false, ";"),
+ GLASS_IDENTIFICATION("examples/src/main/resources/datasets/glass_identification.csv", false, ";"),
/** The Wine recognition data. Could be found <a href="https://archive.ics.uci.edu/ml/machine-learning-databases/wine/">here</a>. */
- WINE_RECOGNITION("modules/ml/src/main/resources/datasets/wine.txt", false, ","),
+ WINE_RECOGNITION("examples/src/main/resources/datasets/wine.txt", false, ","),
/** The Boston house-prices dataset. Could be found <a href="https://archive.ics.uci.edu/ml/machine-learning-databases/housing/">here</a>. */
- BOSTON_HOUSE_PRICES("modules/ml/src/main/resources/datasets/boston_housing_dataset.txt", false, ","),
+ BOSTON_HOUSE_PRICES("examples/src/main/resources/datasets/boston_housing_dataset.txt", false, ","),
/** Example from book Barber D. Bayesian reasoning and machine learning. Chapter 10. */
- ENGLISH_VS_SCOTTISH("modules/ml/src/main/resources/datasets/english_vs_scottish_binary_dataset.csv", true, ","),
+ ENGLISH_VS_SCOTTISH("examples/src/main/resources/datasets/english_vs_scottish_binary_dataset.csv", true, ","),
/** Wholesale customers dataset. Could be found <a href="https://archive.ics.uci.edu/ml/datasets/Wholesale+customers">here</a>. */
- WHOLESALE_CUSTOMERS("modules/ml/src/main/resources/datasets/wholesale_customers.csv", true, ","),
+ WHOLESALE_CUSTOMERS("examples/src/main/resources/datasets/wholesale_customers.csv", true, ","),
/** Fraud detection problem [part of whole dataset]. Could be found <a href="https://www.kaggle.com/mlg-ulb/creditcardfraud/">here</a>. */
- FRAUD_DETECTION("modules/ml/src/main/resources/datasets/fraud_detection.csv", false, ","),
+ FRAUD_DETECTION("examples/src/main/resources/datasets/fraud_detection.csv", false, ","),
/** A dataset with discrete and continuous features. */
- MIXED_DATASET("modules/ml/src/main/resources/datasets/mixed_dataset.csv", true, ","),
+ MIXED_DATASET("examples/src/main/resources/datasets/mixed_dataset.csv", true, ","),
/** A dataset with categorical features and labels. */
- MUSHROOMS("modules/ml/src/main/resources/datasets/mushrooms.csv", true, ",");
+ MUSHROOMS("examples/src/main/resources/datasets/mushrooms.csv", true, ",");
/** Filename. */
private final String filename;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/util/SandboxMLCache.java b/examples/src/main/java/org/apache/ignite/examples/ml/util/SandboxMLCache.java
similarity index 98%
rename from modules/ml/src/main/java/org/apache/ignite/ml/util/SandboxMLCache.java
rename to examples/src/main/java/org/apache/ignite/examples/ml/util/SandboxMLCache.java
index 81e38e2..341a341 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/util/SandboxMLCache.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/util/SandboxMLCache.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.ignite.ml.util;
+package org.apache.ignite.examples.ml.util;
import java.io.File;
import java.io.FileNotFoundException;
@@ -68,7 +68,6 @@
return cache;
}
-
/**
* Loads dataset as a list of rows.
*
@@ -95,11 +94,9 @@
String row = scanner.nextLine();
res.add(row);
}
-
return res;
}
-
/**
* Fills cache with data and returns it.
*
@@ -108,7 +105,6 @@
* @throws FileNotFoundException If file not found.
*/
public IgniteCache<Integer, Vector> fillCacheWith(MLSandboxDatasets dataset) throws FileNotFoundException {
-
IgniteCache<Integer, Vector> cache = getCache();
String fileName = dataset.getFileName();
@@ -137,7 +133,7 @@
try{
if(cells[i].isEmpty()) data[i] = Double.NaN;
else data[i] = Double.valueOf(cells[i]);
- } catch (java.lang.NumberFormatException e) {
+ } catch (NumberFormatException e) {
try {
data[i] = format.parse(cells[i]).doubleValue();
}
@@ -148,7 +144,6 @@
cache.put(cnt++, VectorUtils.of(data));
}
return cache;
-
}
/**
@@ -159,7 +154,6 @@
* @throws FileNotFoundException If file not found.
*/
public IgniteCache<Integer, Object[]> fillObjectCacheWithDoubleLabels(MLSandboxDatasets dataset) throws FileNotFoundException {
-
IgniteCache<Integer, Object[]> cache = getCache2();
String fileName = dataset.getFileName();
@@ -190,7 +184,6 @@
cache.put(cnt++, res);
}
return cache;
-
}
/**
@@ -201,7 +194,6 @@
* @throws FileNotFoundException If file not found.
*/
public IgniteCache<Integer, Object[]> fillObjectCacheWithCategoricalData(MLSandboxDatasets dataset) throws FileNotFoundException {
-
IgniteCache<Integer, Object[]> cache = getCache2();
String fileName = dataset.getFileName();
@@ -225,7 +217,6 @@
cache.put(cnt++, cells);
}
return cache;
-
}
/**
@@ -234,7 +225,6 @@
* @return Filled Ignite Cache.
*/
private IgniteCache<Integer, Vector> getCache() {
-
CacheConfiguration<Integer, Vector> cacheConfiguration = new CacheConfiguration<>();
cacheConfiguration.setName("ML_EXAMPLE_" + UUID.randomUUID());
cacheConfiguration.setAffinity(new RendezvousAffinityFunction(false, 10));
@@ -248,7 +238,6 @@
* @return Filled Ignite Cache.
*/
private IgniteCache<Integer, Object[]> getCache2() {
-
CacheConfiguration<Integer, Object[]> cacheConfiguration = new CacheConfiguration<>();
cacheConfiguration.setName("ML_EXAMPLE_" + UUID.randomUUID());
cacheConfiguration.setAffinity(new RendezvousAffinityFunction(false, 10));
diff --git a/modules/ml/src/main/resources/datasets/boston_housing_dataset.txt b/examples/src/main/resources/datasets/boston_housing_dataset.txt
similarity index 100%
rename from modules/ml/src/main/resources/datasets/boston_housing_dataset.txt
rename to examples/src/main/resources/datasets/boston_housing_dataset.txt
diff --git a/modules/ml/src/main/resources/datasets/cleared_machines.csv b/examples/src/main/resources/datasets/cleared_machines.csv
similarity index 100%
rename from modules/ml/src/main/resources/datasets/cleared_machines.csv
rename to examples/src/main/resources/datasets/cleared_machines.csv
diff --git a/modules/ml/src/main/resources/datasets/english_vs_scottish_binary_dataset.csv b/examples/src/main/resources/datasets/english_vs_scottish_binary_dataset.csv
similarity index 100%
rename from modules/ml/src/main/resources/datasets/english_vs_scottish_binary_dataset.csv
rename to examples/src/main/resources/datasets/english_vs_scottish_binary_dataset.csv
diff --git a/modules/ml/src/main/resources/datasets/fraud_detection.csv b/examples/src/main/resources/datasets/fraud_detection.csv
similarity index 100%
rename from modules/ml/src/main/resources/datasets/fraud_detection.csv
rename to examples/src/main/resources/datasets/fraud_detection.csv
diff --git a/modules/ml/src/main/resources/datasets/glass_identification.csv b/examples/src/main/resources/datasets/glass_identification.csv
similarity index 100%
rename from modules/ml/src/main/resources/datasets/glass_identification.csv
rename to examples/src/main/resources/datasets/glass_identification.csv
diff --git a/modules/ml/src/main/resources/datasets/iris.txt b/examples/src/main/resources/datasets/iris.txt
similarity index 100%
rename from modules/ml/src/main/resources/datasets/iris.txt
rename to examples/src/main/resources/datasets/iris.txt
diff --git a/modules/ml/src/main/resources/datasets/mixed_dataset.csv b/examples/src/main/resources/datasets/mixed_dataset.csv
similarity index 100%
rename from modules/ml/src/main/resources/datasets/mixed_dataset.csv
rename to examples/src/main/resources/datasets/mixed_dataset.csv
diff --git a/modules/ml/src/main/resources/datasets/mortalitydata.csv b/examples/src/main/resources/datasets/mortalitydata.csv
similarity index 100%
rename from modules/ml/src/main/resources/datasets/mortalitydata.csv
rename to examples/src/main/resources/datasets/mortalitydata.csv
diff --git a/modules/ml/src/main/resources/datasets/mushrooms.csv b/examples/src/main/resources/datasets/mushrooms.csv
similarity index 100%
rename from modules/ml/src/main/resources/datasets/mushrooms.csv
rename to examples/src/main/resources/datasets/mushrooms.csv
diff --git a/modules/ml/src/main/resources/datasets/ratings.csv b/examples/src/main/resources/datasets/ratings.csv
similarity index 100%
rename from modules/ml/src/main/resources/datasets/ratings.csv
rename to examples/src/main/resources/datasets/ratings.csv
diff --git a/examples/src/main/resources/datasets/t10k-images-idx3-ubyte b/examples/src/main/resources/datasets/t10k-images-idx3-ubyte
deleted file mode 100644
index 261057a..0000000
--- a/examples/src/main/resources/datasets/t10k-images-idx3-ubyte
+++ /dev/null
Binary files differ
diff --git a/examples/src/main/resources/datasets/t10k-labels-idx1-ubyte b/examples/src/main/resources/datasets/t10k-labels-idx1-ubyte
deleted file mode 100644
index d1c3a97..0000000
--- a/examples/src/main/resources/datasets/t10k-labels-idx1-ubyte
+++ /dev/null
Binary files differ
diff --git a/examples/src/main/resources/datasets/titanic.csv b/examples/src/main/resources/datasets/titanic.csv
index 6994016..ed25e85 100644
--- a/examples/src/main/resources/datasets/titanic.csv
+++ b/examples/src/main/resources/datasets/titanic.csv
@@ -1,17 +1,17 @@
pclass;survived;name;sex;age;sibsp;parch;ticket;fare;cabin;embarked;boat;body;homedest
-1;1;Allen, Miss. Elisabeth Walton;;29;;;24160;211,3375;B5;;2;;St Louis, MO
-1;1;Allison, Master. Hudson Trevor;male;0,9167;1;2;113781;151,55;C22 C26;AA;11;;Montreal, PQ / Chesterville, ON
-1;0;Allison, Miss. Helen Loraine;female;2;1;2;113781;151,55;C22 C26;S;;;Montreal, PQ / Chesterville, ON
-1;0;Allison, Mr. Hudson Joshua Creighton;male;30;1;2;113781;151,55;C22 C26;S;;135;Montreal, PQ / Chesterville, ON
-1;0;Allison, Mrs. Hudson J C (Bessie Waldo Daniels);female;25;1;2;113781;151,55;C22 C26;S;;;Montreal, PQ / Chesterville, ON
-1;1;Anderson, Mr. Harry;male;48;0;0;19952;26,55;E12;S;3;;New York, NY
-1;1;Andrews, Miss. Kornelia Theodosia;female;63;1;0;13502;77,9583;D7;S;10;;Hudson, NY
-1;0;Andrews, Mr. Thomas Jr;male;39;0;0;112050;0;A36;S;;;Belfast, NI
-1;1;Appleton, Mrs. Edward Dale (Charlotte Lamson);female;53;2;0;11769;51,4792;C101;S;D;;Bayside, Queens, NY
-1;0;Artagaveytia, Mr. Ramon;male;71;0;0;PC 17609;49,5042;;C;;22;Montevideo, Uruguay
-1;0;Astor, Col. John Jacob;male;47;1;0;PC 17757;227,525;C62 C64;C;;124;New York, NY
-1;1;Astor, Mrs. John Jacob (Madeleine Talmadge Force);female;18;1;0;PC 17757;227,525;C62 C64;C;4;;New York, NY
-1;1;Aubart, Mme. Leontine Pauline;female;24;0;0;PC 17477;69,3;B35;C;9;;Paris, France
+1;1;Allen ,Miss. Elisabeth Walton;;29;;;24160;211 ,3375;B5;;2;;St Louis ,MO
+1;1;Allison ,Master. Hudson Trevor;male;0 ,9167;1;2;113781;151 ,55;C22 C26;AA;11;;Montreal,PQ / Chesterville,ON
+1;0;Allison ,Miss. Helen Loraine;female;2;1;2;113781;151 ,55;C22 C26;S;;;Montreal ,PQ / Chesterville ,ON
+1;0;Allison ,Mr. Hudson Joshua Creighton;male;30;1;2;113781;151 ,55;C22 C26;S;;135;Montreal ,PQ / Chesterville ,ON
+1;0;Allison ,Mrs. Hudson J C (Bessie Waldo Daniels);female;25;1;2;113781;151 ,55;C22 C26;S;;;Montreal ,PQ / Chesterville ,ON
+1;1;Anderson ,Mr. Harry;male;48;0;0;19952;26 ,55;E12;S;3;;New York ,NY
+1;1;Andrews ,Miss. Kornelia Theodosia;female;63;1;0;13502;77 ,9583;D7;S;10;;Hudson ,NY
+1;0;Andrews ,Mr. Thomas Jr;male;39;0;0;112050;0;A36;S;;;Belfast ,NI
+1;1;Appleton ,Mrs. Edward Dale (Charlotte Lamson);female;53;2;0;11769;51 ,4792;C101;S;D;;Bayside ,Queens ,NY
+1;0;Artagaveytia ,Mr. Ramon;male;71;0;0;PC 17609;49 ,5042;;C;;22;Montevideo ,Uruguay
+1;0;Astor ,Col. John Jacob;male;47;1;0;PC 17757;227 ,525;C62 C64;C;;124;New York,NY
+1;1;Astor ,Mrs. John Jacob (Madeleine Talmadge Force);female;18;1;0;PC 17757;227,525;C62 C64;C;4;;New York ,NY
+1;1;Aubart ,Mme. Leontine Pauline;female;24;0;0;PC 17477;69 ,3;B35;C;9;;Paris ,France
1;1;"Barber, Miss. Ellen ""Nellie""";female;26;0;0;19877;78,85;;S;6;;
1;1;Barkworth, Mr. Algernon Henry Wilson;male;80;0;0;27042;30;A23;S;B;;Hessle, Yorks
1;0;Baumann, Mr. John D;male;;0;0;PC 17318;25,925;;S;;;New York, NY
diff --git a/modules/ml/src/main/resources/datasets/two_classed_iris.csv b/examples/src/main/resources/datasets/two_classed_iris.csv
similarity index 100%
rename from modules/ml/src/main/resources/datasets/two_classed_iris.csv
rename to examples/src/main/resources/datasets/two_classed_iris.csv
diff --git a/modules/ml/src/main/resources/datasets/wholesale_customers.csv b/examples/src/main/resources/datasets/wholesale_customers.csv
similarity index 100%
rename from modules/ml/src/main/resources/datasets/wholesale_customers.csv
rename to examples/src/main/resources/datasets/wholesale_customers.csv
diff --git a/modules/ml/src/main/resources/datasets/wine.txt b/examples/src/main/resources/datasets/wine.txt
similarity index 100%
rename from modules/ml/src/main/resources/datasets/wine.txt
rename to examples/src/main/resources/datasets/wine.txt
diff --git a/examples/src/main/resources/models/mleap/airbnb.model.rf.zip b/examples/src/main/resources/models/mleap/airbnb.model.rf.zip
deleted file mode 100644
index 0da815c..0000000
--- a/examples/src/main/resources/models/mleap/airbnb.model.rf.zip
+++ /dev/null
Binary files differ
diff --git a/examples/src/main/resources/models/mnist_tf_model/saved_model.pb b/examples/src/main/resources/models/mnist_tf_model/saved_model.pb
deleted file mode 100644
index 4d36671..0000000
--- a/examples/src/main/resources/models/mnist_tf_model/saved_model.pb
+++ /dev/null
Binary files differ
diff --git a/examples/src/main/resources/models/mnist_tf_model/variables/variables.data-00000-of-00001 b/examples/src/main/resources/models/mnist_tf_model/variables/variables.data-00000-of-00001
deleted file mode 100644
index a65398f..0000000
--- a/examples/src/main/resources/models/mnist_tf_model/variables/variables.data-00000-of-00001
+++ /dev/null
Binary files differ
diff --git a/examples/src/main/resources/models/mnist_tf_model/variables/variables.index b/examples/src/main/resources/models/mnist_tf_model/variables/variables.index
deleted file mode 100644
index 221dd2d..0000000
--- a/examples/src/main/resources/models/mnist_tf_model/variables/variables.index
+++ /dev/null
Binary files differ
diff --git a/examples/src/main/spark/org/apache/ignite/examples/ml/mleap/MLeapModelParserExample.java b/examples/src/main/spark/org/apache/ignite/examples/ml/mleap/MLeapModelParserExample.java
deleted file mode 100644
index 2462bfd..0000000
--- a/examples/src/main/spark/org/apache/ignite/examples/ml/mleap/MLeapModelParserExample.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.mleap;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.inference.Model;
-import org.apache.ignite.ml.inference.builder.AsyncModelBuilder;
-import org.apache.ignite.ml.inference.builder.IgniteDistributedModelBuilder;
-import org.apache.ignite.ml.inference.reader.FileSystemModelReader;
-import org.apache.ignite.ml.inference.reader.ModelReader;
-import org.apache.ignite.ml.math.primitives.vector.NamedVector;
-import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
-import org.apache.ignite.ml.mleap.MLeapModelParser;
-
-/**
- * This example demonstrates how to import MLeap model and use imported model for distributed inference in Apache
- * Ignite.
- */
-public class MLeapModelParserExample {
- /** Test model resource name. */
- private static final String TEST_MODEL_RES = "examples/src/main/resources/models/mleap/airbnb.model.rf.zip";
-
- /** Parser. */
- private static final MLeapModelParser parser = new MLeapModelParser();
-
- /** Run example. */
- public static void main(String... args) throws ExecutionException, InterruptedException {
- try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- File mdlRsrc = IgniteUtils.resolveIgnitePath(TEST_MODEL_RES);
- if (mdlRsrc == null)
- throw new IllegalArgumentException("File not found [resource_path=" + TEST_MODEL_RES + "]");
-
- ModelReader reader = new FileSystemModelReader(mdlRsrc.getPath());
-
- AsyncModelBuilder mdlBuilder = new IgniteDistributedModelBuilder(ignite, 4, 4);
-
- try (Model<NamedVector, Future<Double>> mdl = mdlBuilder.build(reader, parser)) {
- HashMap<String, Double> input = new HashMap<>();
- input.put("bathrooms", 1.0);
- input.put("bedrooms", 1.0);
- input.put("security_deposit", 1.0);
- input.put("cleaning_fee", 1.0);
- input.put("extra_people", 1.0);
- input.put("number_of_reviews", 1.0);
- input.put("square_feet", 1.0);
- input.put("review_scores_rating", 1.0);
-
- Future<Double> prediction = mdl.predict(VectorUtils.of(input));
-
- System.out.println("Predicted price: " + prediction.get());
- }
- }
- }
-}
diff --git a/modules/ml/mleap-model-parser/pom.xml b/modules/ml/mleap-model-parser/pom.xml
deleted file mode 100644
index 48df94a..0000000
--- a/modules/ml/mleap-model-parser/pom.xml
+++ /dev/null
@@ -1,75 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!--
- POM file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>ignite-parent</artifactId>
- <groupId>org.apache.ignite</groupId>
- <version>1</version>
- <relativePath>../../../parent</relativePath>
- </parent>
-
- <artifactId>ignite-ml-mleap-model-parser</artifactId>
- <version>2.9.0-SNAPSHOT</version>
- <url>http://ignite.apache.org</url>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-libs</id>
- <phase>package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <excludeTransitive>false</excludeTransitive>
- <excludeArtifactIds>
- fastutil,h2,ignite-shmem,annotations,arpack_combined_all,
- cache-api,commons-math3,commons-rng-client-api,commons-rng-core,
- commons-rng-simple,core,ignite-ml
- </excludeArtifactIds>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-ml</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>ml.combust.mleap</groupId>
- <artifactId>mleap-runtime_2.11</artifactId>
- <version>0.13.0</version>
- </dependency>
- </dependencies>
-</project>
diff --git a/modules/ml/mleap-model-parser/src/main/java/org/apache/ignite/ml/mleap/MLeapModel.java b/modules/ml/mleap-model-parser/src/main/java/org/apache/ignite/ml/mleap/MLeapModel.java
deleted file mode 100644
index fe33a5f..0000000
--- a/modules/ml/mleap-model-parser/src/main/java/org/apache/ignite/ml/mleap/MLeapModel.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.mleap;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import ml.combust.mleap.core.types.ScalarType;
-import ml.combust.mleap.core.types.StructField;
-import ml.combust.mleap.core.types.StructType;
-import ml.combust.mleap.runtime.frame.DefaultLeapFrame;
-import ml.combust.mleap.runtime.frame.Row;
-import ml.combust.mleap.runtime.frame.Transformer;
-import ml.combust.mleap.runtime.javadsl.LeapFrameBuilder;
-import org.apache.ignite.ml.inference.Model;
-import org.apache.ignite.ml.math.primitives.vector.NamedVector;
-import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
-import scala.collection.immutable.Set;
-import scala.collection.immutable.Stream;
-import scala.util.Try;
-
-/**
- * MLeap model imported and wrapped to be compatible with Apache Ignite infrastructure.
- */
-public class MLeapModel implements Model<NamedVector, Double> {
- /** MLeap model (transformer in terms of MLeap). */
- private final Transformer transformer;
-
- /** List of field names. */
- private final List<String> schema;
-
- /** Output field name. */
- private final String outputFieldName;
-
- /**
- * Constructs a new instance of MLeap model.
- *
- * @param transformer MLpeap model (transformer in terms of MLeap).
- * @param schema List of field names.
- * @param outputFieldName Output field name.
- */
- public MLeapModel(Transformer transformer, List<String> schema, String outputFieldName) {
- this.transformer = transformer;
- this.schema = new ArrayList<>(schema);
- this.outputFieldName = outputFieldName;
- }
-
- /** {@inheritDoc} */
- @Override public Double predict(NamedVector input) {
- LeapFrameBuilder builder = new LeapFrameBuilder();
- List<StructField> structFields = new ArrayList<>();
-
- List<Object> values = new ArrayList<>();
- for (String fieldName : input.getKeys()) {
- structFields.add(new StructField(fieldName, ScalarType.Double()));
- values.add(input.get(fieldName));
- }
-
- StructType schema = builder.createSchema(structFields);
-
- List<Row> rows = new ArrayList<>();
- rows.add(builder.createRowFromIterable(values));
-
- DefaultLeapFrame inputFrame = builder.createFrame(schema, rows);
-
- return predict(inputFrame);
- }
-
- /**
- * Makes a prediction using default column order specified in {@link #schema}.
- *
- * @param input Input arguments.
- * @return Prediction result.
- */
- public double predict(Double[] input) {
- if (input.length != schema.size())
- throw new IllegalArgumentException("Input size is not equal to schema size");
-
- Map<String, Double> vec = IntStream.range(0, input.length)
- .boxed()
- .collect(Collectors.toMap(schema::get, i -> input[i]));
-
- return predict(VectorUtils.of(vec));
- }
-
- /**
- * Makes a prediction using MLeap API.
- *
- * @param inputFrame Input MLeap frame.
- * @return Prediction result.
- */
- public double predict(DefaultLeapFrame inputFrame) {
- DefaultLeapFrame outputFrame = transformer.transform(inputFrame).get();
-
- Try<DefaultLeapFrame> resFrame = outputFrame.select(new Set.Set1<>(outputFieldName).toSeq());
- DefaultLeapFrame frame = resFrame.get();
-
- Stream<?> stream = (Stream<?>)frame.productElement(1);
- Row row = (Row)stream.head();
-
- return (Double)row.get(0);
- }
-
- /** {@inheritDoc} */
- @Override public void close() {
- transformer.close();
- }
-}
diff --git a/modules/ml/mleap-model-parser/src/main/java/org/apache/ignite/ml/mleap/MLeapModelParser.java b/modules/ml/mleap-model-parser/src/main/java/org/apache/ignite/ml/mleap/MLeapModelParser.java
deleted file mode 100644
index 7f7c258..0000000
--- a/modules/ml/mleap-model-parser/src/main/java/org/apache/ignite/ml/mleap/MLeapModelParser.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.mleap;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import ml.combust.mleap.core.types.ScalarType;
-import ml.combust.mleap.core.types.StructField;
-import ml.combust.mleap.core.types.StructType;
-import ml.combust.mleap.runtime.MleapContext;
-import ml.combust.mleap.runtime.frame.Transformer;
-import ml.combust.mleap.runtime.javadsl.BundleBuilder;
-import ml.combust.mleap.runtime.javadsl.ContextBuilder;
-import ml.combust.mleap.runtime.transformer.PipelineModel;
-import org.apache.ignite.ml.inference.parser.ModelParser;
-import org.apache.ignite.ml.math.primitives.vector.NamedVector;
-import scala.collection.JavaConverters;
-
-/**
- * MLeap model parser.
- */
-public class MLeapModelParser implements ModelParser<NamedVector, Double, MLeapModel> {
- /** */
- private static final long serialVersionUID = -370352744966205715L;
-
- /** Temporary file prefix. */
- private static final String TMP_FILE_PREFIX = "mleap_model";
-
- /** Temporary file postfix. */
- private static final String TMP_FILE_POSTFIX = ".zip";
-
- /** {@inheritDoc} */
- @Override public MLeapModel parse(byte[] mdl) {
- MleapContext mleapCtx = new ContextBuilder().createMleapContext();
- BundleBuilder bundleBuilder = new BundleBuilder();
-
- File file = null;
- try {
- file = File.createTempFile(TMP_FILE_PREFIX, TMP_FILE_POSTFIX);
- try (FileOutputStream fos = new FileOutputStream(file)) {
- fos.write(mdl);
- fos.flush();
- }
-
- Transformer transformer = bundleBuilder.load(file, mleapCtx).root();
- PipelineModel pipelineMdl = (PipelineModel)transformer.model();
-
- List<String> inputSchema = checkAndGetInputSchema(pipelineMdl);
- String outputSchema = checkAndGetOutputSchema(pipelineMdl);
-
- return new MLeapModel(transformer, inputSchema, outputSchema);
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
- finally {
- if (file != null)
- file.delete();
- }
- }
-
- /**
- * Util method that checks that input schema contains only one double type.
- *
- * @param mdl Pipeline model.
- * @return Name of output field.
- */
- private String checkAndGetOutputSchema(PipelineModel mdl) {
- Transformer lastTransformer = mdl.transformers().last();
- StructType outputSchema = lastTransformer.outputSchema();
-
- List<StructField> output = new ArrayList<>(JavaConverters.seqAsJavaListConverter(outputSchema.fields()).asJava());
-
- if (output.size() != 1)
- throw new IllegalArgumentException("Parser supports only scalar outputs");
-
- return output.get(0).name();
- }
-
- /**
- * Util method that checks that output schema contains only double types and returns list of field names.
- *
- * @param mdl Pipeline model.
- * @return List of field names.
- */
- private List<String> checkAndGetInputSchema(PipelineModel mdl) {
- Transformer firstTransformer = mdl.transformers().head();
- StructType inputSchema = firstTransformer.inputSchema();
-
- List<StructField> input = new ArrayList<>(JavaConverters.seqAsJavaListConverter(inputSchema.fields()).asJava());
-
- List<String> schema = new ArrayList<>();
-
- for (StructField field : input) {
- String fieldName = field.name();
-
- schema.add(field.name());
- if (!ScalarType.Double().base().equals(field.dataType().base()))
- throw new IllegalArgumentException("Parser supports only double types [name=" +
- fieldName + ",type=" + field.dataType() + "]");
- }
-
- return schema;
- }
-}
diff --git a/modules/ml/mleap-model-parser/src/main/java/org/apache/ignite/ml/mleap/package-info.java b/modules/ml/mleap-model-parser/src/main/java/org/apache/ignite/ml/mleap/package-info.java
deleted file mode 100644
index 1930ed7..0000000
--- a/modules/ml/mleap-model-parser/src/main/java/org/apache/ignite/ml/mleap/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Base package for Mleap model parser.
- */
-
-package org.apache.ignite.ml.mleap;
diff --git a/modules/ml/mleap-model-parser/src/test/java/org/apache/ignite/ml/mleap/IgniteMLeapTestSuite.java b/modules/ml/mleap-model-parser/src/test/java/org/apache/ignite/ml/mleap/IgniteMLeapTestSuite.java
deleted file mode 100644
index 109b91d..0000000
--- a/modules/ml/mleap-model-parser/src/test/java/org/apache/ignite/ml/mleap/IgniteMLeapTestSuite.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.mleap;
-
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-
-/** Test suite for all module tests. */
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
- MLeapModelParserTest.class
-})
-public class IgniteMLeapTestSuite {
- // No-op.
-}
diff --git a/modules/ml/mleap-model-parser/src/test/java/org/apache/ignite/ml/mleap/MLeapModelParserTest.java b/modules/ml/mleap-model-parser/src/test/java/org/apache/ignite/ml/mleap/MLeapModelParserTest.java
deleted file mode 100644
index 12f6ee1..0000000
--- a/modules/ml/mleap-model-parser/src/test/java/org/apache/ignite/ml/mleap/MLeapModelParserTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.mleap;
-
-import java.net.URL;
-import java.util.HashMap;
-import org.apache.ignite.ml.inference.builder.SingleModelBuilder;
-import org.apache.ignite.ml.inference.builder.SyncModelBuilder;
-import org.apache.ignite.ml.inference.reader.FileSystemModelReader;
-import org.apache.ignite.ml.inference.reader.ModelReader;
-import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Tests for {@link MLeapModelParser}.
- */
-public class MLeapModelParserTest {
- /** Test model resource name. */
- private static final String TEST_MODEL_RESOURCE = "datasets/scikit-airbnb.rf.zip";
-
- /** Parser. */
- private final MLeapModelParser parser = new MLeapModelParser();
-
- /** Model builder. */
- private final SyncModelBuilder mdlBuilder = new SingleModelBuilder();
-
- /** */
- @Test
- public void testParseAndPredict() {
- URL url = MLeapModelParserTest.class.getClassLoader().getResource(TEST_MODEL_RESOURCE);
- if (url == null)
- throw new IllegalStateException("File not found [resource_name=" + TEST_MODEL_RESOURCE + "]");
-
- ModelReader reader = new FileSystemModelReader(url.getPath());
-
- try (MLeapModel mdl = mdlBuilder.build(reader, parser)) {
- HashMap<String, Double> input = new HashMap<>();
- input.put("imp_bathrooms", 1.0);
- input.put("imp_bedrooms", 1.0);
- input.put("imp_security_deposit", 1.0);
- input.put("imp_cleaning_fee", 1.0);
- input.put("imp_extra_people", 1.0);
- input.put("imp_number_of_reviews", 1.0);
- input.put("imp_square_feet", 1.0);
- input.put("imp_review_scores_rating", 1.0);
-
- double prediction = mdl.predict(VectorUtils.of(input));
-
- assertEquals(95.3919, prediction, 1e-5);
- }
- }
-}
diff --git a/modules/ml/mleap-model-parser/src/test/resources/datasets/scikit-airbnb.rf.zip b/modules/ml/mleap-model-parser/src/test/resources/datasets/scikit-airbnb.rf.zip
deleted file mode 100644
index e24d3b4..0000000
--- a/modules/ml/mleap-model-parser/src/test/resources/datasets/scikit-airbnb.rf.zip
+++ /dev/null
Binary files differ
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/Chromosome.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/Chromosome.java
deleted file mode 100644
index 0552036..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/Chromosome.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.Arrays;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-
-/**
- * Represents a potential solution consisting of a fixed-length collection of genes. <br/>
- *
- * <p>
- *
- * NOTE: Chromosome resides in cache: GAGridConstants.POPULATION_CACHE. This cached is partitioned.
- *
- * </p>
- */
-public class Chromosome {
- /** primary key of Chromosome */
- private static final AtomicLong ID_GEN = new AtomicLong();
-
- /** fitness score */
- @QuerySqlField(index = true)
- private double fitnessScore = -1;
-
- /** Id (indexed). */
- @QuerySqlField(index = true)
- private Long id;
-
- /** array of gene keys. */
- private long[] genes;
-
- /**
- * @param genes Primary keys of Genes
- */
- public Chromosome(long[] genes) {
- id = ID_GEN.incrementAndGet();
- this.genes = genes;
- }
-
- /**
- * Gets the fitnessScore
- *
- * @return This chromosome's fitness score
- */
- public double getFitnessScore() {
- return fitnessScore;
- }
-
- /**
- * Set the fitnessScore for this chromosome
- *
- * @param fitnessScore This chromosome's new fitness score
- */
- public void setFitnessScore(double fitnessScore) {
- this.fitnessScore = fitnessScore;
- }
-
- /**
- * Gets the gene keys (ie: primary keys) for this chromosome
- *
- * @return This chromosome's genes
- */
- public long[] getGenes() {
- return genes;
- }
-
- /**
- * Set the gene keys (ie: primary keys)
- *
- * @param genes This chromosome's new genes
- */
- public void setGenes(long[] genes) {
- this.genes = genes;
- }
-
- /**
- * Get the id (primary key) for this chromosome
- *
- * @return This chromosome's primary key
- */
- public Long id() {
- return id;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "Chromosome [fitnessScore=" + fitnessScore + ", id=" + id + ", genes=" + Arrays.toString(genes) + "]";
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/CrossOverJob.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/CrossOverJob.java
deleted file mode 100644
index afd0119..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/CrossOverJob.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.Arrays;
-import java.util.Random;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.compute.ComputeJobAdapter;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.resources.LoggerResource;
-import org.apache.ignite.transactions.Transaction;
-
-/**
- * Responsible for performing 'crossover' genetic operation for 2 X 'parent' chromosomes.
- *
- * <p>
- *
- * It relies on the GAConfiguration.getCrossOverRate() to determine probability rate of crossover for pair of
- * chromosome.
- *
- * <br/>
- *
- * CrossOverJob will randomly pick a start index j in Chromosome.getGenes[] and continue
- *
- * swapping until end of genes[] array.
- *
- * </p>
- */
-public class CrossOverJob extends ComputeJobAdapter {
- /** Ignite resource */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** Ignite logger */
- @LoggerResource
- private IgniteLogger log;
-
- /** primary key of 1st chromosome */
- private Long key1;
-
- /** primary key of 2nd chromosome */
- private Long key2;
-
- /** Cross over rate */
- private double crossOverRate;
-
- /**
- * @param key1 Primary key for 1st chromosome
- * @param key2 Primary key for 2nd chromosome
- * @param crossOverRate CrossOver rate
- */
- public CrossOverJob(Long key1, Long key2, double crossOverRate) {
- this.key1 = key1;
- this.key2 = key2;
- this.crossOverRate = crossOverRate;
- }
-
- /**
- * helper routine to assist cross over
- *
- * @param newKeySwapArrForChrome New gene keys to copy starting at updateIdx
- * @param updateIdx Update Index
- * @param genekeys Original gene Keys for a chromosome
- * @return New Gene keys
- */
- private long[] crossOver(long[] newKeySwapArrForChrome, int updateIdx, long[] genekeys) {
- long[] newGeneKeys = genekeys.clone();
-
- int k = 0;
- for (int x = updateIdx; x < newGeneKeys.length; x++) {
- newGeneKeys[x] = newKeySwapArrForChrome[k];
- k += 1;
- }
- return newGeneKeys;
- }
-
- /** {@inheritDoc} */
- @Override public Object execute() throws IgniteException {
-
- if (this.crossOverRate > Math.random()) {
-
- IgniteCache<Long, Chromosome> populationCache = ignite.cache(GAGridConstants.POPULATION_CACHE);
-
- Transaction tx = ignite.transactions().txStart();
-
- Chromosome chromosome1 = populationCache.localPeek(this.key1);
- Chromosome chromosome2 = populationCache.localPeek(this.key2);
-
- long[] genesforChrom1 = chromosome1.getGenes();
- long[] genesforChrom2 = chromosome2.getGenes();
-
- Random rn = new Random();
-
- // compute index to start for copying respective genes
- int geneIdxStartSwap = rn.nextInt(genesforChrom1.length);
-
- long[] newKeySwapArrForChrome1 =
- Arrays.copyOfRange(genesforChrom2, geneIdxStartSwap, genesforChrom1.length);
- long[] newKeySwapArrForChrome2 =
- Arrays.copyOfRange(genesforChrom1, geneIdxStartSwap, genesforChrom1.length);
-
- long[] newGeneKeysForChrom1 = crossOver(newKeySwapArrForChrome1, geneIdxStartSwap, genesforChrom1);
- long[] newGeneKeysForChrom2 = crossOver(newKeySwapArrForChrome2, geneIdxStartSwap, genesforChrom2);
-
- chromosome1.setGenes(newGeneKeysForChrom1);
- populationCache.put(chromosome1.id(), chromosome1);
-
- chromosome2.setGenes(newGeneKeysForChrom2);
- populationCache.put(chromosome2.id(), chromosome2);
-
- tx.commit();
-
- }
-
- return null;
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/CrossOverTask.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/CrossOverTask.java
deleted file mode 100644
index 7b9ec8c..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/CrossOverTask.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.compute.ComputeJob;
-import org.apache.ignite.compute.ComputeJobResult;
-import org.apache.ignite.compute.ComputeJobResultPolicy;
-import org.apache.ignite.compute.ComputeTaskAdapter;
-import org.apache.ignite.ml.genetic.parameter.GAConfiguration;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * Responsible for assigning 2 X 'parent' chromosomes to produce 2 X 'child' chromosomes.
- *
- * <p>
- *
- * CrossOverTask leverages Ignite's data affinity capabilities for routing CrossOverJobs to primary <br/> IgniteNode
- * where 'parent' chromosomes reside.
- *
- * </p>
- */
-public class CrossOverTask extends ComputeTaskAdapter<List<Long>, Boolean> {
- /** Ignite instance */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** GAConfiguration */
- private GAConfiguration cfg;
-
- /**
- * @param cfg GAConfiguration
- */
- public CrossOverTask(GAConfiguration cfg) {
- this.cfg = cfg;
- }
-
- /** {@inheritDoc} */
- @NotNull @Override public Map map(List<ClusterNode> nodes, List<Long> chromosomeKeys) throws IgniteException {
-
- Map<ComputeJob, ClusterNode> map = new HashMap<>();
-
- Affinity affinity = ignite.affinity(GAGridConstants.POPULATION_CACHE);
-
- Map<ClusterNode, Collection<Long>> nodeKeys = affinity.mapKeysToNodes(chromosomeKeys);
-
- for (Map.Entry<ClusterNode, Collection<Long>> entry : nodeKeys.entrySet()) {
- ClusterNode aNode = entry.getKey();
- map = setupCrossOver(aNode, (List<Long>)entry.getValue(), map);
- }
- return map;
- }
-
- /** {@inheritDoc} */
- @Override public Boolean reduce(List<ComputeJobResult> list) throws IgniteException {
- // TODO Auto-generated method stub
- return Boolean.TRUE;
- }
-
- /** {@inheritDoc} */
- @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) {
- IgniteException err = res.getException();
-
- if (err != null)
- return ComputeJobResultPolicy.FAILOVER;
-
- // If there is no exception, wait for all job results.
- return ComputeJobResultPolicy.WAIT;
-
- }
-
- /**
- * Helper method to help assign ComputeJobs to respective ClusterNodes.
- *
- * @param clusterNode Cluster node.
- * @param keys Primary keys of Chromosomes.
- * @param map Nodes where jobs will be sent.
- * @return A map of ComputeJob/ClusterNode's.
- */
- private Map<ComputeJob, ClusterNode> setupCrossOver(ClusterNode clusterNode, List<Long> keys,
- Map<ComputeJob, ClusterNode> map) {
- // Calculate number of Jobs = keys / 2
- // as we desire pairs of Chromosomes to be swapped
- int numOfJobs = keys.size() / 2;
- int k = 0;
- for (int i = 0; i < numOfJobs; i++) {
- Long key1 = keys.get(k);
- Long key2 = keys.get(k + 1);
-
- CrossOverJob job = new CrossOverJob(key1, key2, this.cfg.getCrossOverRate());
- map.put(job, clusterNode);
- k += 2;
- }
- return map;
- }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/FitnessJob.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/FitnessJob.java
deleted file mode 100644
index 445f1d7..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/FitnessJob.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.compute.ComputeJobAdapter;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.resources.LoggerResource;
-import org.apache.ignite.transactions.Transaction;
-
-/**
- * Responsible for performing fitness evaluation on an individual chromosome
- */
-public class FitnessJob extends ComputeJobAdapter {
- /**
- * Chromosome primary Key
- */
- private Long key;
-
- /** Ignite instance */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** Ignite logger */
- @LoggerResource
- private IgniteLogger log;
-
- /** IFitnessFunction */
- private IFitnessFunction fitnessFunction;
-
- /**
- * @param key Chromosome primary Key
- * @param fitnessFunction Fitness function defined by developer
- */
- public FitnessJob(Long key, IFitnessFunction fitnessFunction) {
- this.key = key;
- this.fitnessFunction = fitnessFunction;
- }
-
- /** {@inheritDoc} */
- @Override public Double execute() throws IgniteException {
-
- IgniteCache<Long, Chromosome> populationCache = ignite.cache(GAGridConstants.POPULATION_CACHE);
-
- IgniteCache<Long, Gene> geneCache = ignite.cache(GAGridConstants.GENE_CACHE);
-
- Chromosome chromosome = populationCache.localPeek(key);
-
- long[] geneKeys = chromosome.getGenes();
-
- List<Gene> genes = new ArrayList<>();
-
- for (int i = 0; i < geneKeys.length; i++) {
- long aKey = geneKeys[i];
- Gene aGene = geneCache.localPeek(aKey);
- genes.add(aGene);
- }
-
- Double val = fitnessFunction.evaluate(genes);
-
- chromosome.setFitnessScore(val);
-
- Transaction tx = ignite.transactions().txStart();
-
- populationCache.put(chromosome.id(), chromosome);
-
- tx.commit();
-
- return val;
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/FitnessTask.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/FitnessTask.java
deleted file mode 100644
index 97b5275..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/FitnessTask.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.compute.ComputeJob;
-import org.apache.ignite.compute.ComputeJobResult;
-import org.apache.ignite.compute.ComputeJobResultPolicy;
-import org.apache.ignite.compute.ComputeTaskAdapter;
-import org.apache.ignite.ml.genetic.parameter.GAConfiguration;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * Responsible for fitness operation
- */
-public class FitnessTask extends ComputeTaskAdapter<List<Long>, Boolean> {
- /** Ignite instance */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** GAConfiguration */
- private GAConfiguration cfg;
-
- /**
- * @param cfg GAConfiguration
- */
- public FitnessTask(GAConfiguration cfg) {
- this.cfg = cfg;
- }
-
- /** {@inheritDoc} */
- @NotNull @Override public Map map(List<ClusterNode> nodes, List<Long> chromosomeKeys) throws IgniteException {
-
- Map<ComputeJob, ClusterNode> map = new HashMap<>();
-
- Affinity affinity = ignite.affinity(GAGridConstants.POPULATION_CACHE);
-
- for (Long key : chromosomeKeys) {
-
- FitnessJob ajob = new FitnessJob(key, this.cfg.getFitnessFunction());
-
- ClusterNode primary = affinity.mapKeyToNode(key);
-
- map.put(ajob, primary);
- }
- return map;
- }
-
- /** {@inheritDoc} */
- @Override public Boolean reduce(List<ComputeJobResult> list) throws IgniteException {
-
- return Boolean.TRUE;
- }
-
- /** {@inheritDoc} */
- @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) {
- IgniteException err = res.getException();
-
- if (err != null)
- return ComputeJobResultPolicy.FAILOVER;
-
- // If there is no exception, wait for all job results.
- return ComputeJobResultPolicy.WAIT;
-
- }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/GAGrid.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/GAGrid.java
deleted file mode 100644
index 68c6f76..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/GAGrid.java
+++ /dev/null
@@ -1,449 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Random;
-import java.util.stream.Collectors;
-import javax.cache.Cache.Entry;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.SqlFieldsQuery;
-import org.apache.ignite.cache.query.SqlQuery;
-import org.apache.ignite.ml.genetic.cache.GeneCacheConfig;
-import org.apache.ignite.ml.genetic.cache.PopulationCacheConfig;
-import org.apache.ignite.ml.genetic.parameter.GAConfiguration;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-
-/**
- * Central class responsible for orchestrating distributive Genetic Algorithm.
- *
- * This class accepts a GAConfigriation and Ignite instance.
- */
-public class GAGrid {
- /** Ignite logger */
- private IgniteLogger igniteLog;
-
- /** GAConfiguraton */
- private GAConfiguration cfg;
-
- /** Ignite instance */
- private Ignite ignite;
-
- /** Population cache */
- private IgniteCache<Long, Chromosome> populationCache;
-
- /** Gene cache */
- private IgniteCache<Long, Gene> geneCache;
-
- /**
- * @param cfg GAConfiguration
- * @param ignite Ignite
- */
- public GAGrid(GAConfiguration cfg, Ignite ignite) {
- this.ignite = ignite;
- this.cfg = cfg;
- this.ignite = ignite;
- this.igniteLog = ignite.log();
-
- // Get/Create cache
- populationCache = this.ignite.getOrCreateCache(PopulationCacheConfig.populationCache());
- populationCache.clear();
-
- // Get/Create cache
- geneCache = this.ignite.getOrCreateCache(GeneCacheConfig.geneCache());
- geneCache.clear();
- }
-
- /**
- * Calculate average fitness value
- *
- * @return Average fitness score
- */
- private Double calculateAverageFitness() {
- double avgFitnessScore = 0;
-
- IgniteCache<Long, Gene> cache = ignite.cache(GAGridConstants.POPULATION_CACHE);
-
- // Execute query calculate average fitness
- SqlFieldsQuery sql = new SqlFieldsQuery("select AVG(FITNESSSCORE) from Chromosome");
-
- // Iterate over the result set.
- try (QueryCursor<List<?>> cursor = cache.query(sql)) {
- for (List<?> row : cursor)
- avgFitnessScore = (Double)row.get(0);
- }
-
- return avgFitnessScore;
- }
-
- /**
- * Calculate fitness each Chromosome in population
- *
- * @param chromosomeKeys List of chromosome primary keys
- */
- private void calculateFitness(List<Long> chromosomeKeys) {
- this.ignite.compute().execute(new FitnessTask(this.cfg), chromosomeKeys);
- }
-
- /**
- * @param fittestKeys List of chromosome keys that will be copied from
- * @param selectedKeys List of chromosome keys that will be overwritten evenly by fittestKeys
- * @return Boolean value
- */
- private Boolean copyFitterChromosomesToPopulation(List<Long> fittestKeys, List<Long> selectedKeys) {
- double truncatePercentage = this.cfg.getTruncateRate();
-
- int totalSize = this.cfg.getPopulationSize();
-
- int truncateCnt = (int)(truncatePercentage * totalSize);
-
- int numOfCopies = selectedKeys.size() / truncateCnt;
-
- return this.ignite.compute()
- .execute(new TruncateSelectionTask(fittestKeys, numOfCopies), selectedKeys);
- }
-
- /**
- * create a Chromsome
- *
- * @param numOfGenes Number of Genes in resepective Chromosome
- * @return Chromosome
- */
- private Chromosome createChromosome(int numOfGenes) {
- long[] genes = new long[numOfGenes];
- List<Long> keys = new ArrayList<>();
- int k = 0;
- while (k < numOfGenes) {
- long key = selectGene(k);
-
- if (!(keys.contains(key))) {
- genes[k] = key;
- keys.add(key);
- k += 1;
- }
- }
- return new Chromosome(genes);
- }
-
- /**
- * Perform crossover
- *
- * @param leastFitKeys List of primary keys for Chromosomes that are considered 'least fit'
- */
- private void crossover(List<Long> leastFitKeys) {
- this.ignite.compute().execute(new CrossOverTask(this.cfg), leastFitKeys);
- }
-
- /**
- * Evolve the population
- *
- * @return Fittest Chromosome
- */
- public Chromosome evolve() {
- // keep track of current generation
- int generationCnt = 1;
-
- Chromosome fittestChromosome;
-
- initializeGenePopulation();
-
- initializePopulation();
-
- // Calculate Fitness
- calculateFitness(getPopulationKeys());
-
- // Retrieve chromosomes in order by fitness value
- LinkedHashMap<Long, Double> map = getChromosomesByFittest();
-
- // Calculate average fitness value of population
- double averageFitnessScore = calculateAverageFitness();
-
- Long key = map.keySet().iterator().next();
-
- fittestChromosome = populationCache.get(key);
-
- // while NOT terminateCondition met
- while (!(cfg.getTerminateCriteria().isTerminationConditionMet(fittestChromosome, averageFitnessScore,
- generationCnt))) {
- generationCnt += 1;
-
- // We will crossover/mutate over chromosomes based on selection method
- List<Long> selectedKeystoreCrossMutation = selection(map);
-
- // Cross Over
- crossover(selectedKeystoreCrossMutation);
-
- // Mutate
- mutation(selectedKeystoreCrossMutation);
-
- // Calculate Fitness
- calculateFitness(selectedKeystoreCrossMutation);
-
- // Retrieve chromosomes in order by fitness value
- map = getChromosomesByFittest();
-
- key = map.keySet().iterator().next();
-
- // Retreive the first chromosome from the list
- fittestChromosome = populationCache.get(key);
-
- // Calculate average fitness value of population
- averageFitnessScore = calculateAverageFitness();
-
- // End Loop
-
- }
- return fittestChromosome;
- }
-
- /**
- * helper routine to retrieve Chromosome keys in order of fittest
- *
- * @return Map of primary key/fitness score pairs for chromosomes.
- */
- private LinkedHashMap<Long,Double> getChromosomesByFittest() {
- LinkedHashMap<Long, Double> orderChromKeysByFittest = new LinkedHashMap<>();
-
- String orderDirection = "desc";
-
- if (!cfg.isHigherFitnessValFitter())
- orderDirection = "asc";
-
- String fittestSQL = "select _key, fitnessScore from Chromosome order by fitnessScore " + orderDirection;
-
- // Execute query to retrieve keys for ALL Chromosomes by fittnessScore
- QueryCursor<List<?>> cursor = populationCache.query(new SqlFieldsQuery(fittestSQL));
-
- List<List<?>> res = cursor.getAll();
-
- for (List row : res) {
- Long key = (Long)row.get(0);
- Double fitnessScore = (Double)row.get(1);
- orderChromKeysByFittest.put(key, fitnessScore);
- }
-
- return orderChromKeysByFittest;
- }
-
- /**
- * @param keys List of primary keys for respective Chromosomes
- * @return List of keys for respective Chromosomes
- */
- private List<Long> getFittestKeysForTruncation(List<Long> keys) {
- double truncatePercentage = this.cfg.getTruncateRate();
-
- int truncateCnt = (int)(truncatePercentage * keys.size());
-
- return keys.subList(0, truncateCnt);
- }
-
- /**
- * initialize the Gene pool
- */
- void initializeGenePopulation() {
- geneCache.clear();
-
- List<Gene> genePool = cfg.getGenePool();
-
- for (Gene gene : genePool)
- geneCache.put(gene.id(), gene);
- }
-
- /**
- * Initialize the population of Chromosomes
- */
- void initializePopulation() {
- int populationSize = cfg.getPopulationSize();
- populationCache.clear();
-
- for (int j = 0; j < populationSize; j++) {
- Chromosome chromosome = createChromosome(cfg.getChromosomeLen());
- populationCache.put(chromosome.id(), chromosome);
- }
-
- }
-
- /**
- * Perform mutation
- *
- * @param leastFitKeys List of primary keys for Chromosomes that are considered 'least fit'.
- */
- private void mutation(List<Long> leastFitKeys) {
- this.ignite.compute().execute(new MutateTask(this.cfg), leastFitKeys);
- }
-
- /**
- * select a gene from the Gene pool
- *
- * @return Primary key of respective Gene
- */
- private long selectAnyGene() {
- int idx = selectRandomIndex(cfg.getGenePool().size());
- Gene gene = cfg.getGenePool().get(idx);
- return gene.id();
- }
-
- /**
- * For our implementation we consider 'best fit' chromosomes, by selecting least fit chromosomes for crossover and
- * mutation
- *
- * As result, we are interested in least fit chromosomes.
- *
- * @param keys List of primary keys for respective Chromosomes
- * @return List of primary Keys for respective Chromosomes that are considered least fit
- */
- private List<Long> selectByElitism(List<Long> keys) {
- int elitismCnt = this.cfg.getElitismCnt();
- return keys.subList(elitismCnt, keys.size());
- }
-
- /**
- * Truncation selection simply retains the fittest x% of the population. These fittest individuals are duplicated so
- * that the population size is maintained.
- *
- * @param keys Keys.
- * @return List of keys
- */
- private List<Long> selectByTruncation(List<Long> keys) {
- double truncatePercentage = this.cfg.getTruncateRate();
-
- int truncateCnt = (int)(truncatePercentage * keys.size());
-
- return keys.subList(truncateCnt, keys.size());
- }
-
- /**
- * Roulette Wheel selection
- *
- * @param map Map of keys/fitness scores
- * @return List of primary Keys for respective chromosomes that will breed
- */
- private List<Long> selectByRouletteWheel(LinkedHashMap map) {
- List<Long> populationKeys = this.ignite.compute().execute(new RouletteWheelSelectionTask(this.cfg), map);
-
- return populationKeys;
- }
-
- /**
- * @param k Gene index in Chromosome.
- * @return Primary key of respective Gene chosen
- */
- private long selectGene(int k) {
- if (cfg.getChromosomeCriteria() == null)
- return (selectAnyGene());
- else
- return (selectGeneByChromosomeCriteria(k));
- }
-
- /**
- * method assumes ChromosomeCriteria is set.
- *
- * @param k Gene index in Chromosome
- * @return Primary key of respective Gene
- */
- private long selectGeneByChromosomeCriteria(int k) {
- List<Gene> genes = new ArrayList<>();
-
- StringBuffer sbSqlClause = new StringBuffer("_val like '");
- sbSqlClause.append("%");
- sbSqlClause.append(cfg.getChromosomeCriteria().getCriteria().get(k));
- sbSqlClause.append("%'");
-
- IgniteCache<Long, Gene> cache = ignite.cache(GAGridConstants.GENE_CACHE);
-
- SqlQuery sql = new SqlQuery(Gene.class, sbSqlClause.toString());
-
- try (QueryCursor<Entry<Long, Gene>> cursor = cache.query(sql)) {
- for (Entry<Long, Gene> e : cursor)
- genes.add(e.getValue());
- }
-
- int idx = selectRandomIndex(genes.size());
-
- Gene gene = genes.get(idx);
- return gene.id();
- }
-
- /**
- * @param sizeOfGenePool Size of Gene pool
- * @return Index
- */
- private int selectRandomIndex(int sizeOfGenePool) {
- Random randomGenerator = new Random();
- return randomGenerator.nextInt(sizeOfGenePool);
- }
-
- /**
- * Select chromosomes
- *
- * @param map Map of keys/fitness scores for respective Chromosomes
- * @return List of primary keys for respective Chromosomes
- */
- private List<Long> selection(LinkedHashMap map) {
- List<Long> selectedKeys = new ArrayList<>();
-
- // We will crossover/mutate over chromosomes based on selection method
- List<Long> chromosomeKeys = new ArrayList<>(map.keySet());
-
- GAGridConstants.SELECTION_METHOD selectionMtd = cfg.getSelectionMtd();
-
- switch (selectionMtd) {
- case SELECTION_METHOD_ELITISM:
- selectedKeys = selectByElitism(chromosomeKeys);
- break;
- case SELECTION_METHOD_TRUNCATION:
- selectedKeys = selectByTruncation(chromosomeKeys);
-
- List<Long> fittestKeys = getFittestKeysForTruncation(chromosomeKeys);
-
- copyFitterChromosomesToPopulation(fittestKeys, selectedKeys);
-
- // copy more fit keys to rest of population
- break;
- case SELECTION_METHOD_ROULETTE_WHEEL:
- selectedKeys = this.selectByRouletteWheel(map);
-
- default:
- break;
- }
-
- return selectedKeys;
- }
-
- /**
- * Get primary keys for Chromosomes
- *
- * @return List of Chromosome primary keys
- */
- List<Long> getPopulationKeys() {
- String fittestSQL = "select _key from Chromosome";
-
- // Execute query to retrieve keys for ALL Chromosomes
- QueryCursor<List<?>> cursor = populationCache.query(new SqlFieldsQuery(fittestSQL));
-
- List<List<?>> res = cursor.getAll();
-
- return (List<Long>) res.stream().map(x -> x.get(0)).collect(Collectors.toList());
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/Gene.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/Gene.java
deleted file mode 100644
index 04b054b..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/Gene.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Represents the discrete parts of a potential solution (ie: Chromosome)
- *
- * <p>
- *
- * Gene is a container for a POJO that developer will implement. <br/>
- *
- * For the Movie Fitness example, the Movie object is the POJO contained within Gene. <br/> NOTE: Gene resides in cache:
- * 'geneCache'. This cache is replicated.
- *
- *
- * </p>
- */
-public class Gene {
- /** primary key of Gene */
- private static final AtomicLong ID_GEN = new AtomicLong();
-
- /** Id (indexed). */
- @QuerySqlField(index = true)
- private Long id;
-
- /** value used to model an individual Gene. */
- private Object val;
-
- /**
- * object Object parameter.
- *
- * @param obj Object parameter.
- */
- public Gene(Object obj) {
- id = ID_GEN.incrementAndGet();
- this.val = obj;
- }
-
- /**
- * @return Value for Gene
- */
- public Object getVal() {
- return val;
- }
-
- /**
- * Set the Gene value
- *
- * @param obj Value for Gene
- */
- public void setVal(Object obj) {
- this.val = obj;
- }
-
- /**
- * @return Primary key for Gene
- */
- public Long id() {
- return id;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "Gene [id=" + id + ", value=" + val + "]";
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/IFitnessFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/IFitnessFunction.java
deleted file mode 100644
index fa84946..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/IFitnessFunction.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.List;
-
-/**
- * Fitness function are used to determine how optimal a particular solution is relative to other solutions.
- *
- * <p> The evaluate() method should be implemented for this interface. The fitness function is provided list of Genes.
- *
- * The evaluate method should return a positive double value that reflects fitness score.
- *
- * </p>
- */
-public interface IFitnessFunction {
- /**
- * @param genes Genes within an individual Chromosome
- * @return Fitness score
- */
- public double evaluate(List<Gene> genes);
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/MutateJob.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/MutateJob.java
deleted file mode 100644
index c3a4d4e..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/MutateJob.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.List;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.compute.ComputeJobAdapter;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.transactions.Transaction;
-
-/**
- * Responsible for applying mutation on respective Chromosome based on mutation Rate
- */
-public class MutateJob extends ComputeJobAdapter {
- /** primary key of Chromosome to mutate **/
- private Long key;
-
- /** primary keys of genes to be used in mutation **/
- private List<Long> mutatedGeneKeys;
-
- /** Ignite instance */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** Mutation Rate **/
- private double mutationRate;
-
- /**
- * @param key Primary key of chromosome
- * @param mutatedGeneKeys Primary keys of genes to be used in mutation
- * @param mutationRate Mutation rate
- */
- public MutateJob(Long key, List<Long> mutatedGeneKeys, double mutationRate) {
- this.key = key;
- this.mutationRate = mutationRate;
- this.mutatedGeneKeys = mutatedGeneKeys;
- }
-
- /** {@inheritDoc} */
- @Override public Boolean execute() throws IgniteException {
-
- IgniteCache<Long, Chromosome> populationCache = ignite.cache(GAGridConstants.POPULATION_CACHE);
-
- Chromosome chromosome = populationCache.localPeek(key);
-
- long[] geneKeys = chromosome.getGenes();
-
- for (int k = 0; k < this.mutatedGeneKeys.size(); k++) {
- // Mutate gene based on MutationRate
- if (this.mutationRate > Math.random())
- geneKeys[k] = this.mutatedGeneKeys.get(k);
- }
-
- chromosome.setGenes(geneKeys);
-
- Transaction tx = ignite.transactions().txStart();
-
- populationCache.put(chromosome.id(), chromosome);
-
- tx.commit();
-
- return Boolean.TRUE;
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/MutateTask.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/MutateTask.java
deleted file mode 100644
index c32406b..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/MutateTask.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import javax.cache.Cache.Entry;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.SqlQuery;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.compute.ComputeJob;
-import org.apache.ignite.compute.ComputeJobResult;
-import org.apache.ignite.compute.ComputeJobResultPolicy;
-import org.apache.ignite.compute.ComputeTaskAdapter;
-import org.apache.ignite.ml.genetic.parameter.GAConfiguration;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * Responsible for applying mutation on respective chromosomes. <br/>
- *
- * MutateTask leverages Ignite's data affinity capabilities for routing MutateJobs to primary IgniteNode where <br/>
- * chromosomes reside.<br/>
- */
-public class MutateTask extends ComputeTaskAdapter<List<Long>, Boolean> {
- /** Ignite instance */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** GAConfiguration */
- private GAConfiguration cfg;
-
- /**
- * @param cfg GAConfiguration
- */
- public MutateTask(GAConfiguration cfg) {
- this.cfg = cfg;
- }
-
- /**
- * choose mutated genes.
- *
- * @return Gene primary keys
- */
- private List<Long> getMutatedGenes() {
- List<Long> mutatedGenes = new ArrayList<>();
- cfg.getChromosomeLen();
-
- for (int i = 0; i < cfg.getChromosomeLen(); i++)
- mutatedGenes.add(selectGene(i));
-
- return mutatedGenes;
- }
-
- /** {@inheritDoc} */
- @NotNull @Override public Map map(List<ClusterNode> nodes, List<Long> chromosomeKeys) throws IgniteException {
-
- Map<ComputeJob, ClusterNode> map = new HashMap<>();
- Affinity affinity = ignite.affinity(GAGridConstants.POPULATION_CACHE);
-
- for (Long key : chromosomeKeys) {
- MutateJob ajob = new MutateJob(key, getMutatedGenes(), this.cfg.getMutationRate());
- ClusterNode primary = affinity.mapKeyToNode(key);
- map.put(ajob, primary);
- }
- return map;
- }
-
- /** {@inheritDoc} */
- @Override public Boolean reduce(List<ComputeJobResult> list) throws IgniteException {
- return Boolean.TRUE;
- }
-
- /** {@inheritDoc} */
- @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) {
- IgniteException err = res.getException();
-
- if (err != null)
- return ComputeJobResultPolicy.FAILOVER;
-
- // If there is no exception, wait for all job results.
- return ComputeJobResultPolicy.WAIT;
-
- }
-
- /**
- * select a gene from the Gene pool
- *
- * @return Primary key of Gene
- */
- private long selectAnyGene() {
- int idx = selectRandomIndex(cfg.getGenePool().size());
- Gene gene = cfg.getGenePool().get(idx);
- return gene.id();
- }
-
- /**
- * select a gene based from the Gene pool
- *
- * @param k Gene index in Chromosome.
- * @return Primary key of Gene
- */
- private long selectGene(int k) {
- if (cfg.getChromosomeCriteria() == null)
- return (selectAnyGene());
- else
- return (selectGeneByChromosomeCriteria(k));
- }
-
- /**
- * method assumes ChromosomeCriteria is set.
- *
- * @param k Gene index in Chromosome.
- * @return Primary key of Gene
- */
- private long selectGeneByChromosomeCriteria(int k) {
- List<Gene> genes = new ArrayList<>();
-
- StringBuffer sbSqlClause = new StringBuffer("_val like '");
- sbSqlClause.append("%");
- sbSqlClause.append(cfg.getChromosomeCriteria().getCriteria().get(k));
- sbSqlClause.append("%'");
-
- IgniteCache<Long, Gene> cache = ignite.cache(GAGridConstants.GENE_CACHE);
-
- SqlQuery sql = new SqlQuery(Gene.class, sbSqlClause.toString());
-
- try (QueryCursor<Entry<Long, Gene>> cursor = cache.query(sql)) {
- for (Entry<Long, Gene> e : cursor)
- genes.add(e.getValue());
- }
-
- int idx = selectRandomIndex(genes.size());
-
- Gene gene = genes.get(idx);
- return gene.id();
- }
-
- /**
- * select an index at random
- *
- * @param sizeOfGenePool Size of gene pool
- * @return Index of Gene to be selected
- */
- private int selectRandomIndex(int sizeOfGenePool) {
- Random randomGenerator = new Random();
- return randomGenerator.nextInt(sizeOfGenePool);
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/RouletteWheelSelectionJob.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/RouletteWheelSelectionJob.java
deleted file mode 100644
index da0085f..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/RouletteWheelSelectionJob.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.stream.Collectors;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.compute.ComputeJobAdapter;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.resources.LoggerResource;
-
-/**
- * Responsible for performing Roulette Wheel selection
- */
-public class RouletteWheelSelectionJob extends ComputeJobAdapter {
- /** Ignite instance */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** Ignite logger */
- @LoggerResource
- private IgniteLogger log;
-
- /** Total Fitness score */
- Double totalFitnessScore;
-
- /** Chromosome key/fitness score pair */
- LinkedHashMap<Long, Double> map;
-
- /**
- * @param totalFitnessScore Total fitness score
- * @param map Chromosome key / fitness score map
- */
- public RouletteWheelSelectionJob(Double totalFitnessScore, LinkedHashMap<Long, Double> map) {
- this.totalFitnessScore = totalFitnessScore;
- this.map = map;
- }
-
- /**
- * Perform Roulette Wheel selection
- *
- * @return Chromosome parent chosen after 'spinning' the wheel.
- */
- @Override public Chromosome execute() throws IgniteException {
-
- IgniteCache<Long, Chromosome> populationCache = ignite.cache(GAGridConstants.POPULATION_CACHE);
-
- int value = spintheWheel(this.totalFitnessScore);
-
- double partialSum = 0;
- boolean notFound = true;
-
- //sort map in ascending order by fitness score
- Map<Long, Double> sortedAscendingMap = map.entrySet().stream()
- .sorted(Map.Entry.comparingByValue())
- .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (e1, e2) -> e1, LinkedHashMap::new));
-
- Iterator<Entry<Long, Double>> entries = sortedAscendingMap.entrySet().iterator();
-
- Long chromosomeKey = (long)-1;
-
- while (entries.hasNext() && notFound) {
- Entry<Long, Double> entry = entries.next();
- Long key = entry.getKey();
- Double fitnessScore = entry.getValue();
- partialSum += fitnessScore;
-
- if (partialSum >= value) {
- notFound = false;
- chromosomeKey = key;
- }
- }
-
- return populationCache.get(chromosomeKey);
- }
-
- /**
- * Spin the wheel.
- *
- * @param fitnessScore Size of Gene pool
- * @return value
- */
- private int spintheWheel(Double fitnessScore) {
- Random randomGenerator = new Random();
- return randomGenerator.nextInt(fitnessScore.intValue());
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/RouletteWheelSelectionTask.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/RouletteWheelSelectionTask.java
deleted file mode 100644
index 74253d8..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/RouletteWheelSelectionTask.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.SqlFieldsQuery;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.compute.ComputeJob;
-import org.apache.ignite.compute.ComputeJobResult;
-import org.apache.ignite.compute.ComputeJobResultPolicy;
-import org.apache.ignite.compute.ComputeLoadBalancer;
-import org.apache.ignite.compute.ComputeTaskAdapter;
-import org.apache.ignite.ml.genetic.parameter.GAConfiguration;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.resources.LoadBalancerResource;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * Responsible for performing Roulette Wheel selection.
- */
-public class RouletteWheelSelectionTask extends ComputeTaskAdapter<LinkedHashMap<Long, Double>, List<Long>> {
- /** Ignite resource. */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** Inject load balancer. */
- @LoadBalancerResource
- ComputeLoadBalancer balancer;
-
- /** GAConfiguration */
- private GAConfiguration cfg;
-
- /**
- * @param cfg GAConfiguration
- */
- public RouletteWheelSelectionTask(GAConfiguration cfg) {
- this.cfg = cfg;
- }
-
- /**
- * Calculate total fitness of population
- *
- * @return Double value representing total fitness score of population
- */
- private Double calculateTotalFitness() {
- double totalFitnessScore = 0;
-
- IgniteCache<Long, Chromosome> cache = ignite.cache(GAGridConstants.POPULATION_CACHE);
-
- SqlFieldsQuery sql = new SqlFieldsQuery("select SUM(FITNESSSCORE) from Chromosome");
-
- // Iterate over the result set.
- try (QueryCursor<List<?>> cursor = cache.query(sql)) {
- for (List<?> row : cursor)
- totalFitnessScore = (Double)row.get(0);
- }
-
- return totalFitnessScore;
- }
-
- /**
- * @param nodes List of ClusterNode.
- * @param chromosomeKeyFitness Map of key/fitness score pairs.
- * @return Map of nodes to jobs.
- */
- @NotNull @Override public Map<ComputeJob, ClusterNode> map(List<ClusterNode> nodes,
- LinkedHashMap<Long, Double> chromosomeKeyFitness) throws IgniteException {
- Map<ComputeJob, ClusterNode> map = new HashMap<>();
-
- Affinity affinity = ignite.affinity(GAGridConstants.POPULATION_CACHE);
- Double totalFitness = this.calculateTotalFitness();
-
- int populationSize = this.cfg.getPopulationSize();
-
- for (int i = 0; i < populationSize; i++) {
- // Pick the next best balanced node for the job.
- RouletteWheelSelectionJob job = new RouletteWheelSelectionJob(totalFitness, chromosomeKeyFitness);
- map.put(job, balancer.getBalancedNode(job, null));
- }
-
- return map;
- }
-
- /**
- * Return list of parent Chromosomes.
- *
- * @param list List of ComputeJobResult.
- * @return List of Chromosome keys.
- */
- @Override public List<Long> reduce(List<ComputeJobResult> list) throws IgniteException {
- List<Chromosome> parents = list.stream().map((x) -> (Chromosome)x.getData()).collect(Collectors.toList());
-
- return createParents(parents);
- }
-
- /**
- * Create new parents and add to populationCache
- *
- * @param parents Chromosomes chosen to breed
- * @return List of Chromosome keys.
- */
- private List<Long> createParents(List<Chromosome> parents) {
- IgniteCache<Long, Chromosome> cache = ignite.cache(GAGridConstants.POPULATION_CACHE);
- cache.clear();
-
- List<Long> keys = new ArrayList();
-
- parents.forEach((x) -> {
- long[] genes = x.getGenes();
- Chromosome newparent = new Chromosome(genes);
- cache.put(newparent.id(), newparent);
- keys.add(newparent.id());
- });
-
- return keys;
- }
-
- /** {@inheritDoc} */
- @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) {
- IgniteException err = res.getException();
-
- if (err != null)
- return ComputeJobResultPolicy.FAILOVER;
-
- // If there is no exception, wait for all job results.
- return ComputeJobResultPolicy.WAIT;
- }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/TruncateSelectionJob.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/TruncateSelectionJob.java
deleted file mode 100644
index 19cbc39..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/TruncateSelectionJob.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.List;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.compute.ComputeJobAdapter;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.transactions.Transaction;
-
-/**
- * Responsible for performing truncate selection
- */
-public class TruncateSelectionJob extends ComputeJobAdapter {
- /** primary key of Chromosome to mutate */
- private Long key;
-
- /** primary keys of genes to be used in mutation */
- private List<Long> mutatedGeneKeys;
-
- /** Ignite instance */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /**
- * @param key Primary key of Chromosome to mutate
- * @param mutatedGeneKeys Primary keys of genes to be used in mutation
- */
- public TruncateSelectionJob(Long key, List<Long> mutatedGeneKeys) {
- this.key = key;
- this.mutatedGeneKeys = mutatedGeneKeys;
- }
-
- /** {@inheritDoc} */
- @Override public Boolean execute() throws IgniteException {
-
- IgniteCache<Long, Chromosome> populationCache = ignite.cache(GAGridConstants.POPULATION_CACHE);
-
- Chromosome chromosome = populationCache.localPeek(key);
-
- long[] geneKeys = chromosome.getGenes();
-
- for (int k = 0; k < this.mutatedGeneKeys.size(); k++)
- geneKeys[k] = this.mutatedGeneKeys.get(k);
-
- chromosome.setGenes(geneKeys);
-
- Transaction tx = ignite.transactions().txStart();
-
- populationCache.put(chromosome.id(), chromosome);
-
- tx.commit();
-
- return Boolean.TRUE;
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/TruncateSelectionTask.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/TruncateSelectionTask.java
deleted file mode 100644
index 9d3ccb1..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/TruncateSelectionTask.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import javax.cache.Cache.Entry;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.SqlQuery;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.compute.ComputeJob;
-import org.apache.ignite.compute.ComputeJobResult;
-import org.apache.ignite.compute.ComputeJobResultPolicy;
-import org.apache.ignite.compute.ComputeTaskAdapter;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * Responsible for performing truncate selection.
- */
-public class TruncateSelectionTask extends ComputeTaskAdapter<List<Long>, Boolean> {
- /** Ignite resource. */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** Fittest keys. */
- private List<Long> fittestKeys;
-
- /** Number of Copies. */
- private int numOfCopies;
-
- /**
- * @param fittestKeys List of long.
- * @param numOfCopies Number of Copies.
- */
- public TruncateSelectionTask(List<Long> fittestKeys, int numOfCopies) {
- this.fittestKeys = fittestKeys;
- this.numOfCopies = numOfCopies;
- }
-
- /**
- * Retrieve a chromosome.
- *
- * @param key Primary key of chromosome.
- * @return Chromosome.
- */
- private Chromosome getChromosome(Long key) {
- IgniteCache<Long, Chromosome> cache = ignite.cache(GAGridConstants.POPULATION_CACHE);
- StringBuffer sbSqlClause = new StringBuffer();
- sbSqlClause.append("_key IN (");
- sbSqlClause.append(key);
- sbSqlClause.append(")");
-
- Chromosome chromosome = null;
-
- SqlQuery sql = new SqlQuery(Chromosome.class, sbSqlClause.toString());
-
- try (QueryCursor<Entry<Long, Chromosome>> cursor = cache.query(sql)) {
- for (Entry<Long, Chromosome> e : cursor)
- chromosome = (e.getValue());
- }
-
- return chromosome;
- }
-
- /**
- * Return a List of lists containing keys.
- *
- * @return List of lists containing keys.
- */
- private List<List<Long>> getEnhancedPopulation() {
- List<List<Long>> list = new ArrayList<>();
-
- for (Long key : fittestKeys) {
- Chromosome cp = getChromosome(key);
- for (int i = 0; i < numOfCopies; i++) {
- long[] thegenes = cp.getGenes();
- List<Long> geneList = new ArrayList<>();
- for (int k = 0; k < cp.getGenes().length; k++)
- geneList.add(thegenes[k]);
-
- list.add(geneList);
- }
- }
-
- return list;
- }
-
- /** {@inheritDoc} */
- @NotNull @Override public Map map(List<ClusterNode> nodes, List<Long> chromosomeKeys) throws IgniteException {
- Map<ComputeJob, ClusterNode> map = new HashMap<>();
- Affinity affinity = ignite.affinity(GAGridConstants.POPULATION_CACHE);
-
- // Retrieve enhanced population
- List<List<Long>> enhancedPopulation = getEnhancedPopulation();
-
- int k = 0;
- for (Long key : chromosomeKeys) {
- TruncateSelectionJob ajob = new TruncateSelectionJob(key, enhancedPopulation.get(k));
- ClusterNode primary = affinity.mapKeyToNode(key);
- map.put(ajob, primary);
- k += 1;
- }
- return map;
- }
-
- /** {@inheritDoc} */
- @Override public Boolean reduce(List<ComputeJobResult> list) throws IgniteException {
- return Boolean.TRUE;
- }
-
- /** {@inheritDoc} */
- @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) {
- IgniteException err = res.getException();
-
- if (err != null)
- return ComputeJobResultPolicy.FAILOVER;
-
- // If there is no exception, wait for all job results.
- return ComputeJobResultPolicy.WAIT;
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/cache/GeneCacheConfig.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/cache/GeneCacheConfig.java
deleted file mode 100644
index f980e22..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/cache/GeneCacheConfig.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic.cache;
-
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.functions.GAGridFunction;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-
-/**
- * Cache configuration for GAGridConstants.GENE_CACHE
- *
- * cache maintains full population of genes.
- */
-public class GeneCacheConfig {
- /**
- * @return Cache Configuration
- */
- public static CacheConfiguration<Long, Gene> geneCache() {
-
- CacheConfiguration<Long, Gene> cfg = new CacheConfiguration<>(GAGridConstants.GENE_CACHE);
- cfg.setIndexedTypes(Long.class, Gene.class);
- cfg.setCacheMode(CacheMode.REPLICATED);
- cfg.setRebalanceMode(CacheRebalanceMode.SYNC);
- cfg.setStatisticsEnabled(true);
- cfg.setBackups(1);
- cfg.setSqlFunctionClasses(GAGridFunction.class);
- return cfg;
-
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/cache/PopulationCacheConfig.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/cache/PopulationCacheConfig.java
deleted file mode 100644
index 6a8b2b4..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/cache/PopulationCacheConfig.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic.cache;
-
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.genetic.Chromosome;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-
-/**
- * Cache configuration for GAGridConstants.POPULATION_CACHE
- *
- * cache population of chromosomes (ie: potential solutions)
- */
-public class PopulationCacheConfig {
- /**
- * @return Cache Configuration
- */
- public static CacheConfiguration<Long, Chromosome> populationCache() {
-
- CacheConfiguration<Long, Chromosome> cfg = new CacheConfiguration<>(GAGridConstants.POPULATION_CACHE);
- cfg.setIndexedTypes(Long.class, Chromosome.class);
- cfg.setCacheMode(CacheMode.PARTITIONED);
- cfg.setRebalanceMode(CacheRebalanceMode.SYNC);
- cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- cfg.setStatisticsEnabled(true);
- cfg.setBackups(1);
-
- return cfg;
-
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/cache/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/cache/package-info.java
deleted file mode 100644
index 3dfd401..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/cache/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Contains cache configurations for GA Grid
- */
-
-package org.apache.ignite.ml.genetic.cache;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/functions/GAGridFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/functions/GAGridFunction.java
deleted file mode 100644
index bdc3baf..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/functions/GAGridFunction.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic.functions;
-
-import java.sql.SQLException;
-import java.sql.Types;
-import java.util.List;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.query.annotations.QuerySqlFunction;
-import org.apache.ignite.ml.genetic.Chromosome;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.utils.GAGridUtils;
-import org.h2.tools.SimpleResultSet;
-
-/**
- * Responsible for providing custom SQL functions to retrieve optimization results.
- */
-public class GAGridFunction {
- /** */
- public GAGridFunction() {
- }
-
- /**
- * Retrieve solutions in descending order based on fitness score.
- *
- * @return Result set.
- * @throws SQLException If failed.
- */
- @QuerySqlFunction
- public static SimpleResultSet getSolutionsDesc() {
- return (getChromosomes("order by fitnessScore desc"));
- }
-
- /**
- * Retrieve solutions in ascending order based on fitness score.
- *
- * @return Result set
- * @throws SQLException If failed.
- */
- @QuerySqlFunction
- public static SimpleResultSet getSolutionsAsc() throws SQLException {
- return (getChromosomes("order by fitnessScore asc"));
- }
-
- /**
- * Retrieve and individual solution by Chromosome key.
- *
- * @param key Primary key of Chromosome.
- * @return SimpleResultSet.
- * @throws SQLException If failed.
- */
- @QuerySqlFunction
- public static SimpleResultSet getSolutionById(int key) throws SQLException {
- StringBuffer sbSqlClause = new StringBuffer();
- sbSqlClause.append("_key IN");
- sbSqlClause.append("(");
- sbSqlClause.append(key);
- sbSqlClause.append(")");
- return (getChromosomes(sbSqlClause.toString()));
- }
-
- /**
- * Helper routine to return 'pivoted' results using the provided query param.
- *
- * @param qry Sql
- * @return Result set
- */
- private static SimpleResultSet getChromosomes(String qry) {
- Ignite ignite = Ignition.localIgnite();
-
- List<Chromosome> chromosomes = GAGridUtils.getChromosomes(ignite, qry);
-
- SimpleResultSet rs2 = new SimpleResultSet();
-
- Chromosome aChrom = chromosomes.get(0);
- int genesCnt = aChrom.getGenes().length;
-
- rs2.addColumn("Chromosome Id", Types.INTEGER, 0, 0);
- rs2.addColumn("Fitness Score", Types.DOUBLE, 0, 0);
-
- for (int i = 0; i < genesCnt; i++) {
- int colIdx = i + 1;
- rs2.addColumn("Gene " + colIdx, Types.VARCHAR, 0, 0);
- }
-
- for (Chromosome rowChrom : chromosomes) {
-
- Object[] row = new Object[genesCnt + 2];
- row[0] = rowChrom.id();
- row[1] = rowChrom.getFitnessScore();
-
- List<Gene> genes = GAGridUtils.getGenesInOrderForChromosome(ignite, rowChrom);
- int i = 2;
-
- for (Gene gene : genes) {
- row[i] = gene.getVal().toString();
- i += 1;
- }
- //Add a row for an individual Chromosome
- rs2.addRow(row);
- }
-
- return rs2;
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/functions/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/functions/package-info.java
deleted file mode 100644
index 8822a00..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/functions/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Contains functions used for GA Grid
- */
-
-package org.apache.ignite.ml.genetic.functions;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/package-info.java
deleted file mode 100644
index fad2a52..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Root GA package (GA Grid)
- */
-
-package org.apache.ignite.ml.genetic;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/ChromosomeCriteria.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/ChromosomeCriteria.java
deleted file mode 100644
index de035be..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/ChromosomeCriteria.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic.parameter;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Responsible for describing the characteristics of an individual Chromosome.
- */
-public class ChromosomeCriteria {
- /** List of criteria for a Chromosome */
- private List<String> criteria = new ArrayList<>();
-
- /**
- * Retrieve criteria
- *
- * @return List of strings
- */
- public List<String> getCriteria() {
- return criteria;
- }
-
- /**
- * Set criteria
- *
- * @param criteria List of criteria to be applied for a Chromosome ;Use format "name=value", ie: "coinType=QUARTER"
- */
- public void setCriteria(List<String> criteria) {
- this.criteria = criteria;
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/GAConfiguration.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/GAConfiguration.java
deleted file mode 100644
index c7da056..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/GAConfiguration.java
+++ /dev/null
@@ -1,332 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic.parameter;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.IFitnessFunction;
-
-/**
- * Maintains configuration parameters to be used in genetic algorithm
- *
- * <br/>
- *
- * <p>
- *
- * NOTE: Default selectionMethod is SELECTION_METHOD_TRUNCATION
- *
- * Default truncateRate is .10
- *
- * More selectionMethods will be introduced in future releases.
- *
- * </p>
- */
-public class GAConfiguration {
- /** Selection method */
- private GAGridConstants.SELECTION_METHOD selectionMtd;
-
- /** Criteria used to describe a chromosome */
- private ChromosomeCriteria chromosomeCriteria;
-
- /**
- * Percentage of most fit chromosomes to be maintained and utilized to copy into new population.
- *
- * NOTE: This parameter is only considered when selectionMethod is SELECTION_METHOD_TRUNCATION
- *
- * Accepted values between 0 and 1
- */
- private double truncateRate;
-
- /**
- * Elitism is the concept that the strongest members of the population will be preserved from generation to
- * generation. <br/>
- *
- * No crossovers or mutations will be performed for elite chromosomes.
- *
- * NOTE: This parameter is only considered when selectionMethod is SELECTION_METHOD_ELITISM.
- */
- private int elitismCnt;
-
- /**
- * Indicates how chromosome fitness values should be evaluated. </br> A chromosome with
- * isHigherFitnessValueFitter=true is considered fittest.
- */
- private boolean isHigherFitnessValFitter = true;
-
- /**
- * Population size represents the number of potential solutions (ie: chromosomes) between each generation Default
- * size is 500
- *
- * </br> NOTE: The population size remains fixed between each generation
- */
- private int populationSize = 500;
-
- /** Gene pool is the sum of ALL genes utilized to create chromosomes */
- private List<Gene> genePool = new ArrayList<>();
-
- /** Number of genes within a chromosome */
- private int chromosomeLen;
-
- /**
- * Crossover rate is the probability that two chromosomes will breed with each other. offspring with traits of each
- * of the parents.
- *
- * Accepted values are between 0 and 1
- */
- private double crossOverRate = .50;
-
- /**
- * Mutation rate is the probability that a chromosome will be mutated offspring with traits of each of the parents.
- *
- *
- * Accepted values are between 0 and 1
- */
- private double mutationRate = .50;
-
- /**
- * Call back interface used to terminate Genetic algorithm.
- *
- * Implement this interface based on particular use case.
- */
- private ITerminateCriteria terminateCriteria;
-
- /**
- * Represents a fitness function. Implement the IFitnessFunction to satisfy your particular use case.
- */
- private IFitnessFunction fitnessFunction;
-
- /**
- * Default constructor.
- */
- public GAConfiguration() {
- this.setSelectionMtd(GAGridConstants.SELECTION_METHOD.SELECTION_METHOD_TRUNCATION);
- this.setTruncateRate(.10);
- }
-
- /**
- * retrieve the ChromosomeCriteria
- *
- * @return Chromosome criteria
- */
- public ChromosomeCriteria getChromosomeCriteria() {
- return chromosomeCriteria;
- }
-
- /**
- * set value for ChromosomeCriteria
- *
- * @param chromosomeCriteria Chromosome criteria
- */
-
- public void setChromosomeCriteria(ChromosomeCriteria chromosomeCriteria) {
- this.chromosomeCriteria = chromosomeCriteria;
- }
-
- /**
- * @return Boolean value indicating how fitness values should be evaluated.
- */
- public boolean isHigherFitnessValFitter() {
- return this.isHigherFitnessValFitter;
- }
-
- /**
- * Retrieve the chromosome length
- *
- * @return Size of Chromosome
- */
- public int getChromosomeLen() {
- return chromosomeLen;
- }
-
- /**
- * Set the Chromsome length
- *
- * @param chromosomeLen Size of Chromosome
- */
- public void setChromosomeLen(int chromosomeLen) {
- this.chromosomeLen = chromosomeLen;
- }
-
- /**
- * Retrieve the cross over rate
- *
- * @return Cross over rate
- */
- public double getCrossOverRate() {
- return crossOverRate;
- }
-
- /**
- * Set the cross over rate.
- *
- * @param crossOverRate Cross over rate
- */
- public void setCrossOverRate(double crossOverRate) {
- this.crossOverRate = crossOverRate;
- }
-
- /**
- * Retrieve the elitism count
- *
- * @return Elitism count
- */
- public int getElitismCnt() {
- return elitismCnt;
- }
-
- /**
- * Set the elitism count.
- *
- * @param elitismCnt Elitism count
- */
- public void setElitismCnt(int elitismCnt) {
- this.elitismCnt = elitismCnt;
- }
-
- /**
- * Retrieve IFitnessFunction
- *
- * @return Fitness function
- */
- public IFitnessFunction getFitnessFunction() {
- return fitnessFunction;
- }
-
- /**
- * Set IFitnessFunction
- *
- * @param fitnessFunction Fitness function
- */
- public void setFitnessFunction(IFitnessFunction fitnessFunction) {
- this.fitnessFunction = fitnessFunction;
- }
-
- /**
- * Retrieve the gene pool
- *
- * @return List of Genes
- */
- public List<Gene> getGenePool() {
- return (this.genePool);
- }
-
- /**
- * Set the gene pool.
- *
- * NOTE: When Apache Ignite is started the gene pool is utilized to initialize the distributed
- * GAGridConstants.GENE_CACHE.
- *
- * @param genePool List of Genes
- */
- public void setGenePool(List<Gene> genePool) {
- this.genePool = genePool;
- }
-
- /**
- * Retrieve the mutation rate.
- *
- * @return Mutation Rate
- */
- public double getMutationRate() {
- return mutationRate;
- }
-
- /**
- * Set the mutation rate.
- *
- * @param mutationRate Mutation Rate
- */
- public void setMutationRate(double mutationRate) {
- this.mutationRate = mutationRate;
- }
-
- /**
- * Retrieve the population size
- *
- * @return Population size
- */
-
- public int getPopulationSize() {
- return populationSize;
- }
-
- /**
- * Set the population size
- *
- * @param populationSize Size of population
- */
- public void setPopulationSize(int populationSize) {
- this.populationSize = populationSize;
- }
-
- /**
- * Get the selection method
- *
- * @return Selection method
- */
- public GAGridConstants.SELECTION_METHOD getSelectionMtd() {
- return selectionMtd;
- }
-
- /**
- * Set the selection method
- *
- * @param selectionMtd Selection method
- */
- public void setSelectionMtd(GAGridConstants.SELECTION_METHOD selectionMtd) {
- this.selectionMtd = selectionMtd;
- }
-
- /**
- * Retrieve the termination criteria
- *
- * @return Termination Criteria
- */
- public ITerminateCriteria getTerminateCriteria() {
- return terminateCriteria;
- }
-
- /**
- * Set the termination criteria.
- *
- * @param terminateCriteria Termination Criteria
- */
- public void setTerminateCriteria(ITerminateCriteria terminateCriteria) {
- this.terminateCriteria = terminateCriteria;
- }
-
- /**
- * Retrieve truncateRate
- *
- * @return Truncate Rate
- */
- public double getTruncateRate() {
- return truncateRate;
- }
-
- /**
- * Set truncatePercentage
- *
- * @param truncateRate Truncate rate
- */
- public void setTruncateRate(double truncateRate) {
- this.truncateRate = truncateRate;
- }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/GAGridConstants.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/GAGridConstants.java
deleted file mode 100644
index 86111ea..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/GAGridConstants.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic.parameter;
-
-/**
- * GAGridConstants
- */
-public interface GAGridConstants {
- /** populationCache constant */
- public static final String POPULATION_CACHE = "populationCache";
-
- /** populationCache constant */
- public static final String GENE_CACHE = "geneCache";
-
- /** Selection Method type **/
- public enum SELECTION_METHOD {
- /** Selection method eletism. */
- SELECTION_METHOD_ELITISM,
- /** Selection method truncation. */
- SELECTION_METHOD_TRUNCATION,
- /** Selection method roulette wheel. */
- SELECTION_METHOD_ROULETTE_WHEEL
- }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/ITerminateCriteria.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/ITerminateCriteria.java
deleted file mode 100644
index 5868b3d..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/ITerminateCriteria.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic.parameter;
-
-import org.apache.ignite.ml.genetic.Chromosome;
-
-/**
- * Represents the terminate condition for a genetic algorithm.
- *
- * <p>
- *
- * Implement this interface for your respective use case.
- *
- * </p>
- */
-public interface ITerminateCriteria {
- /**
- * @param fittestChromosome Fittest chromosome as of the nth generation
- * @param averageFitnessScore Average fitness score
- * @param generation Current number of generations
- * @return Boolean value to determine when to stop evolution
- */
- public boolean isTerminationConditionMet(Chromosome fittestChromosome, double averageFitnessScore, int generation);
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/package-info.java
deleted file mode 100644
index a22dd60..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/parameter/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Contains parameters used for GA Grid
- */
-
-package org.apache.ignite.ml.genetic.parameter;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/utils/GAGridUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/utils/GAGridUtils.java
deleted file mode 100644
index c80fa52..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/utils/GAGridUtils.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic.utils;
-
-import java.util.ArrayList;
-import java.util.List;
-import javax.cache.Cache.Entry;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.SqlQuery;
-import org.apache.ignite.ml.genetic.Chromosome;
-import org.apache.ignite.ml.genetic.Gene;
-import org.apache.ignite.ml.genetic.cache.PopulationCacheConfig;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-
-/**
- * GA Grid Helper routines
- */
-public class GAGridUtils {
- /**
- * Retrieve chromosomes
- *
- * @param ignite Ignite
- * @param qry Sql
- * @return List of Chromosomes
- */
- public static List<Chromosome> getChromosomes(Ignite ignite, String qry) {
- List<Chromosome> chromosomes = new ArrayList<>();
-
- IgniteCache<Long, Chromosome> populationCache = ignite.getOrCreateCache(PopulationCacheConfig.populationCache());
-
- SqlQuery sql = new SqlQuery(Chromosome.class, qry);
-
- try (QueryCursor<Entry<Long, Chromosome>> cursor = populationCache.query(sql)) {
- for (Entry<Long, Chromosome> e : cursor)
- chromosomes.add(e.getValue());
- }
-
- return chromosomes;
- }
-
- /**
- * Retrieve genes in order
- *
- * @param ignite Ignite
- * @param chromosome Chromosome
- * @return List of Genes
- */
- public static List<Gene> getGenesInOrderForChromosome(Ignite ignite, Chromosome chromosome) {
- List<Gene> genes = new ArrayList<>();
- IgniteCache<Long, Gene> cache = ignite.cache(GAGridConstants.GENE_CACHE);
-
- long[] primaryKeys = chromosome.getGenes();
-
- for (int k = 0; k < primaryKeys.length; k++) {
-
- StringBuffer sbSqlClause = new StringBuffer();
- sbSqlClause.append("_key IN ");
- sbSqlClause.append("(");
- sbSqlClause.append(primaryKeys[k]);
- sbSqlClause.append(")");
-
- SqlQuery sql = new SqlQuery(Gene.class, sbSqlClause.toString());
-
- try (QueryCursor<Entry<Long, Gene>> cursor = cache.query(sql)) {
- for (Entry<Long, Gene> e : cursor)
- genes.add(e.getValue());
- }
- }
-
- return genes;
- }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/utils/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/genetic/utils/package-info.java
deleted file mode 100644
index eaf23ba5..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/genetic/utils/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Contains utils for GA Grid
- */
-
-package org.apache.ignite.ml.genetic.utils;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/SerializableConsumer.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDoubleConsumer.java
similarity index 78%
rename from modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/SerializableConsumer.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDoubleConsumer.java
index ece58aa..baa478a 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/SerializableConsumer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDoubleConsumer.java
@@ -15,15 +15,15 @@
* limitations under the License.
*/
-package org.apache.ignite.tensorflow.util;
+package org.apache.ignite.ml.math.functions;
import java.io.Serializable;
-import java.util.function.Consumer;
+import java.util.function.DoubleConsumer;
/**
- * Serializable consumer.
+ * Serializable DoubleConsumer.
*
- * @param <T> The type of the input to the operation.
+ * @see DoubleConsumer
*/
-public interface SerializableConsumer<T> extends Consumer<T>, Serializable {
+public interface IgniteDoubleConsumer extends DoubleConsumer, Serializable {
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/pipeline/Pipeline.java b/modules/ml/src/main/java/org/apache/ignite/ml/pipeline/Pipeline.java
index b869aed..cea87c5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/pipeline/Pipeline.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/pipeline/Pipeline.java
@@ -43,7 +43,7 @@
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
*/
-public class Pipeline<K, V, C extends Serializable, L> {
+public class Pipeline<K, V, C extends Serializable, L> implements Serializable {
/** Final Feature extractor. */
private Preprocessor<K, V> finalPreprocessor;
@@ -158,4 +158,11 @@
.withPreprocessor(finalPreprocessor)
.withInternalMdl(internalMdl);
}
+
+ /**
+ * Returns the final preprocessor for evaluation needs.
+ */
+ public Preprocessor<K, V> getFinalPreprocessor() {
+ return finalPreprocessor;
+ }
}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/selection/cv/AbstractCrossValidation.java b/modules/ml/src/main/java/org/apache/ignite/ml/selection/cv/AbstractCrossValidation.java
index 911be41..518bcef 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/selection/cv/AbstractCrossValidation.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/selection/cv/AbstractCrossValidation.java
@@ -17,18 +17,35 @@
package org.apache.ignite.ml.selection.cv;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import org.apache.ignite.lang.IgniteBiPredicate;
import org.apache.ignite.ml.IgniteModel;
import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.environment.LearningEnvironment;
import org.apache.ignite.ml.environment.LearningEnvironmentBuilder;
import org.apache.ignite.ml.environment.parallelism.Promise;
+import org.apache.ignite.ml.math.functions.IgniteDoubleConsumer;
import org.apache.ignite.ml.math.functions.IgniteSupplier;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.pipeline.Pipeline;
import org.apache.ignite.ml.pipeline.PipelineMdl;
import org.apache.ignite.ml.preprocessing.Preprocessor;
-import org.apache.ignite.ml.selection.paramgrid.*;
+import org.apache.ignite.ml.selection.paramgrid.BruteForceStrategy;
+import org.apache.ignite.ml.selection.paramgrid.EvolutionOptimizationStrategy;
+import org.apache.ignite.ml.selection.paramgrid.HyperParameterTuningStrategy;
+import org.apache.ignite.ml.selection.paramgrid.ParamGrid;
+import org.apache.ignite.ml.selection.paramgrid.ParameterSetGenerator;
+import org.apache.ignite.ml.selection.paramgrid.RandomStrategy;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.Metric;
import org.apache.ignite.ml.selection.scoring.metric.MetricName;
@@ -39,12 +56,6 @@
import org.apache.ignite.ml.util.genetic.GeneticAlgorithm;
import org.jetbrains.annotations.NotNull;
-import java.util.*;
-import java.util.function.BiFunction;
-import java.util.function.DoubleConsumer;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
/**
* Cross validation score calculator. Cross validation is an approach that allows to avoid overfitting that is made the
* following way: the training set is split into k smaller sets. The following procedure is followed for each of the k
@@ -59,7 +70,7 @@
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
*/
-public abstract class AbstractCrossValidation<M extends IgniteModel<Vector, Double>, K, V> {
+public abstract class AbstractCrossValidation<M extends IgniteModel<Vector, Double>, K, V> implements Serializable {
/** Learning environment builder. */
protected LearningEnvironmentBuilder envBuilder = LearningEnvironmentBuilder.defaultBuilder();
@@ -300,7 +311,7 @@
Map<String, Double> paramMap = new HashMap<>();
for (int paramIdx = 0; paramIdx < paramSet.length; paramIdx++) {
- DoubleConsumer setter = paramGrid.getSetterByIndex(paramIdx);
+ IgniteDoubleConsumer setter = paramGrid.getSetterByIndex(paramIdx);
Double paramVal = paramSet[paramIdx];
setter.accept(paramVal);
@@ -330,10 +341,10 @@
return pnt < from || pnt > to;
};
- IgniteBiPredicate<K, V> testSetFilter = (k, v) -> !trainSetFilter.apply(k,v);
+ IgniteBiPredicate<K, V> testSetFilter = (k, v) -> !trainSetFilter.apply(k, v);
DatasetBuilder<K, V> trainSet = datasetBuilderSupplier.apply(trainSetFilter);
- M mdl = trainer.fit(trainSet, preprocessor); //TODO: IGNITE-11580
+ M mdl = trainer.fit(trainSet, preprocessor);
DatasetBuilder<K, V> testSet = datasetBuilderSupplier.apply(testSetFilter);
scores[i] = Evaluator.evaluate(testSet, mdl, preprocessor, metric).getSingle();
@@ -361,13 +372,13 @@
return pnt < from || pnt > to;
};
- IgniteBiPredicate<K, V> testSetFilter = (k, v) -> !trainSetFilter.apply(k,v);
+ IgniteBiPredicate<K, V> testSetFilter = (k, v) -> !trainSetFilter.apply(k, v);
- DatasetBuilder<K, V> datasetBuilder = datasetBuilderSupplier.apply(trainSetFilter);
- PipelineMdl<K, V> mdl = pipeline.fit(datasetBuilder);
+ DatasetBuilder<K, V> trainSet = datasetBuilderSupplier.apply(trainSetFilter);
+ PipelineMdl<K, V> mdl = pipeline.fit(trainSet);
DatasetBuilder<K, V> testSet = datasetBuilderSupplier.apply(testSetFilter);
- scores[i] = Evaluator.evaluate(testSet, mdl, preprocessor, metric).getSingle();
+ scores[i] = Evaluator.evaluate(testSet, mdl, pipeline.getFinalPreprocessor(), metric).getSingle();
}
return scores;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/selection/cv/CrossValidationResult.java b/modules/ml/src/main/java/org/apache/ignite/ml/selection/cv/CrossValidationResult.java
index f7ddae3..ad50cc5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/selection/cv/CrossValidationResult.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/selection/cv/CrossValidationResult.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ml.selection.cv;
+import java.io.Serializable;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -26,7 +27,7 @@
* Represents the cross validation procedure result,
* wraps score and values of hyper parameters associated with these values.
*/
-public class CrossValidationResult {
+public class CrossValidationResult implements Serializable {
/** Best hyper params. */
private Map<String, Double> bestHyperParams;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/selection/paramgrid/ParamGrid.java b/modules/ml/src/main/java/org/apache/ignite/ml/selection/paramgrid/ParamGrid.java
index 11878a1..054ed90 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/selection/paramgrid/ParamGrid.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/selection/paramgrid/ParamGrid.java
@@ -17,22 +17,23 @@
package org.apache.ignite.ml.selection.paramgrid;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.function.DoubleConsumer;
+import org.apache.ignite.ml.math.functions.IgniteDoubleConsumer;
/**
* Keeps the grid of parameters.
*/
-public class ParamGrid {
+public class ParamGrid implements Serializable {
/** Parameter values by parameter index. */
private Map<Integer, Double[]> paramValuesByParamIdx = new HashMap<>();
/** Parameter names by parameter index. */
- private Map<Integer, DoubleConsumer> settersByParamIdx = new HashMap<>();
+ private Map<Integer, IgniteDoubleConsumer> settersByParamIdx = new HashMap<>();
/** Parameter names by parameter index. */
private Map<Integer, String> paramNamesByParamIdx = new HashMap<>();
@@ -55,7 +56,7 @@
* @param params The array of the given hyper parameter values.
* @return The updated ParamGrid.
*/
- public ParamGrid addHyperParam(String paramName, DoubleConsumer setter, Double[] params) {
+ public ParamGrid addHyperParam(String paramName, IgniteDoubleConsumer setter, Double[] params) {
paramValuesByParamIdx.put(paramCntr, params);
paramNamesByParamIdx.put(paramCntr, paramName);
settersByParamIdx.put(paramCntr, setter);
@@ -80,7 +81,7 @@
}
/** Returns setter for parameter with the given index. */
- public DoubleConsumer getSetterByIndex(int idx) {
+ public IgniteDoubleConsumer getSetterByIndex(int idx) {
return settersByParamIdx.get(idx);
}
diff --git a/modules/ml/src/main/resources/datasets/titanic.csv b/modules/ml/src/main/resources/datasets/titanic.csv
deleted file mode 100644
index ed25e85..0000000
--- a/modules/ml/src/main/resources/datasets/titanic.csv
+++ /dev/null
@@ -1,1310 +0,0 @@
-pclass;survived;name;sex;age;sibsp;parch;ticket;fare;cabin;embarked;boat;body;homedest
-1;1;Allen ,Miss. Elisabeth Walton;;29;;;24160;211 ,3375;B5;;2;;St Louis ,MO
-1;1;Allison ,Master. Hudson Trevor;male;0 ,9167;1;2;113781;151 ,55;C22 C26;AA;11;;Montreal,PQ / Chesterville,ON
-1;0;Allison ,Miss. Helen Loraine;female;2;1;2;113781;151 ,55;C22 C26;S;;;Montreal ,PQ / Chesterville ,ON
-1;0;Allison ,Mr. Hudson Joshua Creighton;male;30;1;2;113781;151 ,55;C22 C26;S;;135;Montreal ,PQ / Chesterville ,ON
-1;0;Allison ,Mrs. Hudson J C (Bessie Waldo Daniels);female;25;1;2;113781;151 ,55;C22 C26;S;;;Montreal ,PQ / Chesterville ,ON
-1;1;Anderson ,Mr. Harry;male;48;0;0;19952;26 ,55;E12;S;3;;New York ,NY
-1;1;Andrews ,Miss. Kornelia Theodosia;female;63;1;0;13502;77 ,9583;D7;S;10;;Hudson ,NY
-1;0;Andrews ,Mr. Thomas Jr;male;39;0;0;112050;0;A36;S;;;Belfast ,NI
-1;1;Appleton ,Mrs. Edward Dale (Charlotte Lamson);female;53;2;0;11769;51 ,4792;C101;S;D;;Bayside ,Queens ,NY
-1;0;Artagaveytia ,Mr. Ramon;male;71;0;0;PC 17609;49 ,5042;;C;;22;Montevideo ,Uruguay
-1;0;Astor ,Col. John Jacob;male;47;1;0;PC 17757;227 ,525;C62 C64;C;;124;New York,NY
-1;1;Astor ,Mrs. John Jacob (Madeleine Talmadge Force);female;18;1;0;PC 17757;227,525;C62 C64;C;4;;New York ,NY
-1;1;Aubart ,Mme. Leontine Pauline;female;24;0;0;PC 17477;69 ,3;B35;C;9;;Paris ,France
-1;1;"Barber, Miss. Ellen ""Nellie""";female;26;0;0;19877;78,85;;S;6;;
-1;1;Barkworth, Mr. Algernon Henry Wilson;male;80;0;0;27042;30;A23;S;B;;Hessle, Yorks
-1;0;Baumann, Mr. John D;male;;0;0;PC 17318;25,925;;S;;;New York, NY
-1;0;Baxter, Mr. Quigg Edmond;male;24;0;1;PC 17558;247,5208;B58 B60;C;;;Montreal, PQ
-1;1;Baxter, Mrs. James (Helene DeLaudeniere Chaput);female;50;0;1;PC 17558;247,5208;B58 B60;C;6;;Montreal, PQ
-1;1;Bazzani, Miss. Albina;female;32;0;0;11813;76,2917;D15;C;8;;
-1;0;Beattie, Mr. Thomson;male;36;0;0;13050;75,2417;C6;C;A;;Winnipeg, MN
-1;1;Beckwith, Mr. Richard Leonard;male;37;1;1;11751;52,5542;D35;S;5;;New York, NY
-1;1;Beckwith, Mrs. Richard Leonard (Sallie Monypeny);female;47;1;1;11751;52,5542;D35;S;5;;New York, NY
-1;1;Behr, Mr. Karl Howell;male;26;0;0;111369;30;C148;C;5;;New York, NY
-1;1;Bidois, Miss. Rosalie;female;42;0;0;PC 17757;227,525;;C;4;;
-1;1;Bird, Miss. Ellen;female;29;0;0;PC 17483;221,7792;C97;S;8;;
-1;0;Birnbaum, Mr. Jakob;male;25;0;0;13905;26;;C;;148;San Francisco, CA
-1;1;Bishop, Mr. Dickinson H;male;25;1;0;11967;91,0792;B49;C;7;;Dowagiac, MI
-1;1;Bishop, Mrs. Dickinson H (Helen Walton);female;19;1;0;11967;91,0792;B49;C;7;;Dowagiac, MI
-1;1;Bissette, Miss. Amelia;female;35;0;0;PC 17760;135,6333;C99;S;8;;
-1;1;Bjornstrom-Steffansson, Mr. Mauritz Hakan;male;28;0;0;110564;26,55;C52;S;D;;Stockholm, Sweden / Washington, DC
-1;0;Blackwell, Mr. Stephen Weart;male;45;0;0;113784;35,5;T;S;;;Trenton, NJ
-1;1;Blank, Mr. Henry;male;40;0;0;112277;31;A31;C;7;;Glen Ridge, NJ
-1;1;Bonnell, Miss. Caroline;female;30;0;0;36928;164,8667;C7;S;8;;Youngstown, OH
-1;1;Bonnell, Miss. Elizabeth;female;58;0;0;113783;26,55;C103;S;8;;Birkdale, England Cleveland, Ohio
-1;0;Borebank, Mr. John James;male;42;0;0;110489;26,55;D22;S;;;London / Winnipeg, MB
-1;1;Bowen, Miss. Grace Scott;female;45;0;0;PC 17608;262,375;;C;4;;Cooperstown, NY
-1;1;Bowerman, Miss. Elsie Edith;female;22;0;1;113505;55;E33;S;6;;St Leonards-on-Sea, England Ohio
-1;1;"Bradley, Mr. George (""George Arthur Brayton"")";male;;0;0;111427;26,55;;S;9;;Los Angeles, CA
-1;0;Brady, Mr. John Bertram;male;41;0;0;113054;30,5;A21;S;;;Pomeroy, WA
-1;0;Brandeis, Mr. Emil;male;48;0;0;PC 17591;50,4958;B10;C;;208;Omaha, NE
-1;0;Brewe, Dr. Arthur Jackson;male;;0;0;112379;39,6;;C;;;Philadelphia, PA
-1;1;Brown, Mrs. James Joseph (Margaret Tobin);female;44;0;0;PC 17610;27,7208;B4;C;6;;Denver, CO
-1;1;Brown, Mrs. John Murray (Caroline Lane Lamson);female;59;2;0;11769;51,4792;C101;S;D;;Belmont, MA
-1;1;Bucknell, Mrs. William Robert (Emma Eliza Ward);female;60;0;0;11813;76,2917;D15;C;8;;Philadelphia, PA
-1;1;Burns, Miss. Elizabeth Margaret;female;41;0;0;16966;134,5;E40;C;3;;
-1;0;Butt, Major. Archibald Willingham;male;45;0;0;113050;26,55;B38;S;;;Washington, DC
-1;0;Cairns, Mr. Alexander;male;;0;0;113798;31;;S;;;
-1;1;Calderhead, Mr. Edward Pennington;male;42;0;0;PC 17476;26,2875;E24;S;5;;New York, NY
-1;1;Candee, Mrs. Edward (Helen Churchill Hungerford);female;53;0;0;PC 17606;27,4458;;C;6;;Washington, DC
-1;1;Cardeza, Mr. Thomas Drake Martinez;male;36;0;1;PC 17755;512,3292;B51 B53 B55;C;3;;Austria-Hungary / Germantown, Philadelphia, PA
-1;1;Cardeza, Mrs. James Warburton Martinez (Charlotte Wardle Drake);female;58;0;1;PC 17755;512,3292;B51 B53 B55;C;3;;Germantown, Philadelphia, PA
-1;0;Carlsson, Mr. Frans Olof;male;33;0;0;695;5;B51 B53 B55;S;;;New York, NY
-1;0;Carrau, Mr. Francisco M;male;28;0;0;113059;47,1;;S;;;Montevideo, Uruguay
-1;0;Carrau, Mr. Jose Pedro;male;17;0;0;113059;47,1;;S;;;Montevideo, Uruguay
-1;1;Carter, Master. William Thornton II;male;11;1;2;113760;120;B96 B98;S;4;;Bryn Mawr, PA
-1;1;Carter, Miss. Lucile Polk;female;14;1;2;113760;120;B96 B98;S;4;;Bryn Mawr, PA
-1;1;Carter, Mr. William Ernest;male;36;1;2;113760;120;B96 B98;S;C;;Bryn Mawr, PA
-1;1;Carter, Mrs. William Ernest (Lucile Polk);female;36;1;2;113760;120;B96 B98;S;4;;Bryn Mawr, PA
-1;0;Case, Mr. Howard Brown;male;49;0;0;19924;26;;S;;;Ascot, Berkshire / Rochester, NY
-1;1;Cassebeer, Mrs. Henry Arthur Jr (Eleanor Genevieve Fosdick);female;;0;0;17770;27,7208;;C;5;;New York, NY
-1;0;Cavendish, Mr. Tyrell William;male;36;1;0;19877;78,85;C46;S;;172;Little Onn Hall, Staffs
-1;1;Cavendish, Mrs. Tyrell William (Julia Florence Siegel);female;76;1;0;19877;78,85;C46;S;6;;Little Onn Hall, Staffs
-1;0;Chaffee, Mr. Herbert Fuller;male;46;1;0;W.E.P. 5734;61,175;E31;S;;;Amenia, ND
-1;1;Chaffee, Mrs. Herbert Fuller (Carrie Constance Toogood);female;47;1;0;W.E.P. 5734;61,175;E31;S;4;;Amenia, ND
-1;1;Chambers, Mr. Norman Campbell;male;27;1;0;113806;53,1;E8;S;5;;New York, NY / Ithaca, NY
-1;1;Chambers, Mrs. Norman Campbell (Bertha Griggs);female;33;1;0;113806;53,1;E8;S;5;;New York, NY / Ithaca, NY
-1;1;Chaudanson, Miss. Victorine;female;36;0;0;PC 17608;262,375;B61;C;4;;
-1;1;Cherry, Miss. Gladys;female;30;0;0;110152;86,5;B77;S;8;;London, England
-1;1;Chevre, Mr. Paul Romaine;male;45;0;0;PC 17594;29,7;A9;C;7;;Paris, France
-1;1;Chibnall, Mrs. (Edith Martha Bowerman);female;;0;1;113505;55;E33;S;6;;St Leonards-on-Sea, England Ohio
-1;0;Chisholm, Mr. Roderick Robert Crispin;male;;0;0;112051;0;;S;;;Liverpool, England / Belfast
-1;0;Clark, Mr. Walter Miller;male;27;1;0;13508;136,7792;C89;C;;;Los Angeles, CA
-1;1;Clark, Mrs. Walter Miller (Virginia McDowell);female;26;1;0;13508;136,7792;C89;C;4;;Los Angeles, CA
-1;1;Cleaver, Miss. Alice;female;22;0;0;113781;151,55;;S;11;;
-1;0;Clifford, Mr. George Quincy;male;;0;0;110465;52;A14;S;;;Stoughton, MA
-1;0;Colley, Mr. Edward Pomeroy;male;47;0;0;5727;25,5875;E58;S;;;Victoria, BC
-1;1;Compton, Miss. Sara Rebecca;female;39;1;1;PC 17756;83,1583;E49;C;14;;Lakewood, NJ
-1;0;Compton, Mr. Alexander Taylor Jr;male;37;1;1;PC 17756;83,1583;E52;C;;;Lakewood, NJ
-1;1;Compton, Mrs. Alexander Taylor (Mary Eliza Ingersoll);female;64;0;2;PC 17756;83,1583;E45;C;14;;Lakewood, NJ
-1;1;Cornell, Mrs. Robert Clifford (Malvina Helen Lamson);female;55;2;0;11770;25,7;C101;S;2;;New York, NY
-1;0;Crafton, Mr. John Bertram;male;;0;0;113791;26,55;;S;;;Roachdale, IN
-1;0;Crosby, Capt. Edward Gifford;male;70;1;1;WE/P 5735;71;B22;S;;269;Milwaukee, WI
-1;1;Crosby, Miss. Harriet R;female;36;0;2;WE/P 5735;71;B22;S;7;;Milwaukee, WI
-1;1;Crosby, Mrs. Edward Gifford (Catherine Elizabeth Halstead);female;64;1;1;112901;26,55;B26;S;7;;Milwaukee, WI
-1;0;Cumings, Mr. John Bradley;male;39;1;0;PC 17599;71,2833;C85;C;;;New York, NY
-1;1;Cumings, Mrs. John Bradley (Florence Briggs Thayer);female;38;1;0;PC 17599;71,2833;C85;C;4;;New York, NY
-1;1;Daly, Mr. Peter Denis ;male;51;0;0;113055;26,55;E17;S;5 9;;Lima, Peru
-1;1;Daniel, Mr. Robert Williams;male;27;0;0;113804;30,5;;S;3;;Philadelphia, PA
-1;1;Daniels, Miss. Sarah;female;33;0;0;113781;151,55;;S;8;;
-1;0;Davidson, Mr. Thornton;male;31;1;0;F.C. 12750;52;B71;S;;;Montreal, PQ
-1;1;Davidson, Mrs. Thornton (Orian Hays);female;27;1;2;F.C. 12750;52;B71;S;3;;Montreal, PQ
-1;1;Dick, Mr. Albert Adrian;male;31;1;0;17474;57;B20;S;3;;Calgary, AB
-1;1;Dick, Mrs. Albert Adrian (Vera Gillespie);female;17;1;0;17474;57;B20;S;3;;Calgary, AB
-1;1;Dodge, Dr. Washington;male;53;1;1;33638;81,8583;A34;S;13;;San Francisco, CA
-1;1;Dodge, Master. Washington;male;4;0;2;33638;81,8583;A34;S;5;;San Francisco, CA
-1;1;Dodge, Mrs. Washington (Ruth Vidaver);female;54;1;1;33638;81,8583;A34;S;5;;San Francisco, CA
-1;0;Douglas, Mr. Walter Donald;male;50;1;0;PC 17761;106,425;C86;C;;62;Deephaven, MN / Cedar Rapids, IA
-1;1;Douglas, Mrs. Frederick Charles (Mary Helene Baxter);female;27;1;1;PC 17558;247,5208;B58 B60;C;6;;Montreal, PQ
-1;1;Douglas, Mrs. Walter Donald (Mahala Dutton);female;48;1;0;PC 17761;106,425;C86;C;2;;Deephaven, MN / Cedar Rapids, IA
-1;1;"Duff Gordon, Lady. (Lucille Christiana Sutherland) (""Mrs Morgan"")";female;48;1;0;11755;39,6;A16;C;1;;London / Paris
-1;1;"Duff Gordon, Sir. Cosmo Edmund (""Mr Morgan"")";male;49;1;0;PC 17485;56,9292;A20;C;1;;London / Paris
-1;0;Dulles, Mr. William Crothers;male;39;0;0;PC 17580;29,7;A18;C;;133;Philadelphia, PA
-1;1;Earnshaw, Mrs. Boulton (Olive Potter);female;23;0;1;11767;83,1583;C54;C;7;;Mt Airy, Philadelphia, PA
-1;1;Endres, Miss. Caroline Louise;female;38;0;0;PC 17757;227,525;C45;C;4;;New York, NY
-1;1;Eustis, Miss. Elizabeth Mussey;female;54;1;0;36947;78,2667;D20;C;4;;Brookline, MA
-1;0;Evans, Miss. Edith Corse;female;36;0;0;PC 17531;31,6792;A29;C;;;New York, NY
-1;0;Farthing, Mr. John;male;;0;0;PC 17483;221,7792;C95;S;;;
-1;1;Flegenheim, Mrs. Alfred (Antoinette);female;;0;0;PC 17598;31,6833;;S;7;;New York, NY
-1;1;Fleming, Miss. Margaret;female;;0;0;17421;110,8833;;C;4;;
-1;1;"Flynn, Mr. John Irwin (""Irving"")";male;36;0;0;PC 17474;26,3875;E25;S;5;;Brooklyn, NY
-1;0;Foreman, Mr. Benjamin Laventall;male;30;0;0;113051;27,75;C111;C;;;New York, NY
-1;1;Fortune, Miss. Alice Elizabeth;female;24;3;2;19950;263;C23 C25 C27;S;10;;Winnipeg, MB
-1;1;Fortune, Miss. Ethel Flora;female;28;3;2;19950;263;C23 C25 C27;S;10;;Winnipeg, MB
-1;1;Fortune, Miss. Mabel Helen;female;23;3;2;19950;263;C23 C25 C27;S;10;;Winnipeg, MB
-1;0;Fortune, Mr. Charles Alexander;male;19;3;2;19950;263;C23 C25 C27;S;;;Winnipeg, MB
-1;0;Fortune, Mr. Mark;male;64;1;4;19950;263;C23 C25 C27;S;;;Winnipeg, MB
-1;1;Fortune, Mrs. Mark (Mary McDougald);female;60;1;4;19950;263;C23 C25 C27;S;10;;Winnipeg, MB
-1;1;Francatelli, Miss. Laura Mabel;female;30;0;0;PC 17485;56,9292;E36;C;1;;
-1;0;Franklin, Mr. Thomas Parham;male;;0;0;113778;26,55;D34;S;;;Westcliff-on-Sea, Essex
-1;1;Frauenthal, Dr. Henry William;male;50;2;0;PC 17611;133,65;;S;5;;New York, NY
-1;1;Frauenthal, Mr. Isaac Gerald;male;43;1;0;17765;27,7208;D40;C;5;;New York, NY
-1;1;Frauenthal, Mrs. Henry William (Clara Heinsheimer);female;;1;0;PC 17611;133,65;;S;5;;New York, NY
-1;1;Frolicher, Miss. Hedwig Margaritha;female;22;0;2;13568;49,5;B39;C;5;;Zurich, Switzerland
-1;1;Frolicher-Stehli, Mr. Maxmillian;male;60;1;1;13567;79,2;B41;C;5;;Zurich, Switzerland
-1;1;Frolicher-Stehli, Mrs. Maxmillian (Margaretha Emerentia Stehli);female;48;1;1;13567;79,2;B41;C;5;;Zurich, Switzerland
-1;0;Fry, Mr. Richard;male;;0;0;112058;0;B102;S;;;
-1;0;Futrelle, Mr. Jacques Heath;male;37;1;0;113803;53,1;C123;S;;;Scituate, MA
-1;1;Futrelle, Mrs. Jacques Heath (Lily May Peel);female;35;1;0;113803;53,1;C123;S;D;;Scituate, MA
-1;0;Gee, Mr. Arthur H;male;47;0;0;111320;38,5;E63;S;;275;St Anne's-on-Sea, Lancashire
-1;1;Geiger, Miss. Amalie;female;35;0;0;113503;211,5;C130;C;4;;
-1;1;Gibson, Miss. Dorothy Winifred;female;22;0;1;112378;59,4;;C;7;;New York, NY
-1;1;Gibson, Mrs. Leonard (Pauline C Boeson);female;45;0;1;112378;59,4;;C;7;;New York, NY
-1;0;Giglio, Mr. Victor;male;24;0;0;PC 17593;79,2;B86;C;;;
-1;1;Goldenberg, Mr. Samuel L;male;49;1;0;17453;89,1042;C92;C;5;;Paris, France / New York, NY
-1;1;Goldenberg, Mrs. Samuel L (Edwiga Grabowska);female;;1;0;17453;89,1042;C92;C;5;;Paris, France / New York, NY
-1;0;Goldschmidt, Mr. George B;male;71;0;0;PC 17754;34,6542;A5;C;;;New York, NY
-1;1;Gracie, Col. Archibald IV;male;53;0;0;113780;28,5;C51;C;B;;Washington, DC
-1;1;Graham, Miss. Margaret Edith;female;19;0;0;112053;30;B42;S;3;;Greenwich, CT
-1;0;Graham, Mr. George Edward;male;38;0;1;PC 17582;153,4625;C91;S;;147;Winnipeg, MB
-1;1;Graham, Mrs. William Thompson (Edith Junkins);female;58;0;1;PC 17582;153,4625;C125;S;3;;Greenwich, CT
-1;1;Greenfield, Mr. William Bertram;male;23;0;1;PC 17759;63,3583;D10 D12;C;7;;New York, NY
-1;1;Greenfield, Mrs. Leo David (Blanche Strouse);female;45;0;1;PC 17759;63,3583;D10 D12;C;7;;New York, NY
-1;0;Guggenheim, Mr. Benjamin;male;46;0;0;PC 17593;79,2;B82 B84;C;;;New York, NY
-1;1;Harder, Mr. George Achilles;male;25;1;0;11765;55,4417;E50;C;5;;Brooklyn, NY
-1;1;Harder, Mrs. George Achilles (Dorothy Annan);female;25;1;0;11765;55,4417;E50;C;5;;Brooklyn, NY
-1;1;Harper, Mr. Henry Sleeper;male;48;1;0;PC 17572;76,7292;D33;C;3;;New York, NY
-1;1;Harper, Mrs. Henry Sleeper (Myna Haxtun);female;49;1;0;PC 17572;76,7292;D33;C;3;;New York, NY
-1;0;Harrington, Mr. Charles H;male;;0;0;113796;42,4;;S;;;
-1;0;Harris, Mr. Henry Birkhardt;male;45;1;0;36973;83,475;C83;S;;;New York, NY
-1;1;Harris, Mrs. Henry Birkhardt (Irene Wallach);female;35;1;0;36973;83,475;C83;S;D;;New York, NY
-1;0;Harrison, Mr. William;male;40;0;0;112059;0;B94;S;;110;
-1;1;Hassab, Mr. Hammad;male;27;0;0;PC 17572;76,7292;D49;C;3;;
-1;1;Hawksford, Mr. Walter James;male;;0;0;16988;30;D45;S;3;;Kingston, Surrey
-1;1;Hays, Miss. Margaret Bechstein;female;24;0;0;11767;83,1583;C54;C;7;;New York, NY
-1;0;Hays, Mr. Charles Melville;male;55;1;1;12749;93,5;B69;S;;307;Montreal, PQ
-1;1;Hays, Mrs. Charles Melville (Clara Jennings Gregg);female;52;1;1;12749;93,5;B69;S;3;;Montreal, PQ
-1;0;Head, Mr. Christopher;male;42;0;0;113038;42,5;B11;S;;;London / Middlesex
-1;0;Hilliard, Mr. Herbert Henry;male;;0;0;17463;51,8625;E46;S;;;Brighton, MA
-1;0;Hipkins, Mr. William Edward;male;55;0;0;680;50;C39;S;;;London / Birmingham
-1;1;Hippach, Miss. Jean Gertrude;female;16;0;1;111361;57,9792;B18;C;4;;Chicago, IL
-1;1;Hippach, Mrs. Louis Albert (Ida Sophia Fischer);female;44;0;1;111361;57,9792;B18;C;4;;Chicago, IL
-1;1;Hogeboom, Mrs. John C (Anna Andrews);female;51;1;0;13502;77,9583;D11;S;10;;Hudson, NY
-1;0;Holverson, Mr. Alexander Oskar;male;42;1;0;113789;52;;S;;38;New York, NY
-1;1;Holverson, Mrs. Alexander Oskar (Mary Aline Towner);female;35;1;0;113789;52;;S;8;;New York, NY
-1;1;"Homer, Mr. Harry (""Mr E Haven"")";male;35;0;0;111426;26,55;;C;15;;Indianapolis, IN
-1;1;Hoyt, Mr. Frederick Maxfield;male;38;1;0;19943;90;C93;S;D;;New York, NY / Stamford CT
-1;0;Hoyt, Mr. William Fisher;male;;0;0;PC 17600;30,6958;;C;14;;New York, NY
-1;1;Hoyt, Mrs. Frederick Maxfield (Jane Anne Forby);female;35;1;0;19943;90;C93;S;D;;New York, NY / Stamford CT
-1;1;Icard, Miss. Amelie;female;38;0;0;113572;80;B28;;6;;
-1;0;Isham, Miss. Ann Elizabeth;female;50;0;0;PC 17595;28,7125;C49;C;;;Paris, France New York, NY
-1;1;Ismay, Mr. Joseph Bruce;male;49;0;0;112058;0;B52 B54 B56;S;C;;Liverpool
-1;0;Jones, Mr. Charles Cresson;male;46;0;0;694;26;;S;;80;Bennington, VT
-1;0;Julian, Mr. Henry Forbes;male;50;0;0;113044;26;E60;S;;;London
-1;0;Keeping, Mr. Edwin;male;32,5;0;0;113503;211,5;C132;C;;45;
-1;0;Kent, Mr. Edward Austin;male;58;0;0;11771;29,7;B37;C;;258;Buffalo, NY
-1;0;Kenyon, Mr. Frederick R;male;41;1;0;17464;51,8625;D21;S;;;Southington / Noank, CT
-1;1;Kenyon, Mrs. Frederick R (Marion);female;;1;0;17464;51,8625;D21;S;8;;Southington / Noank, CT
-1;1;Kimball, Mr. Edwin Nelson Jr;male;42;1;0;11753;52,5542;D19;S;5;;Boston, MA
-1;1;Kimball, Mrs. Edwin Nelson Jr (Gertrude Parsons);female;45;1;0;11753;52,5542;D19;S;5;;Boston, MA
-1;0;Klaber, Mr. Herman;male;;0;0;113028;26,55;C124;S;;;Portland, OR
-1;1;Kreuchen, Miss. Emilie;female;39;0;0;24160;211,3375;;S;2;;
-1;1;Leader, Dr. Alice (Farnham);female;49;0;0;17465;25,9292;D17;S;8;;New York, NY
-1;1;LeRoy, Miss. Bertha;female;30;0;0;PC 17761;106,425;;C;2;;
-1;1;Lesurer, Mr. Gustave J;male;35;0;0;PC 17755;512,3292;B101;C;3;;
-1;0;Lewy, Mr. Ervin G;male;;0;0;PC 17612;27,7208;;C;;;Chicago, IL
-1;0;"Lindeberg-Lind, Mr. Erik Gustaf (""Mr Edward Lingrey"")";male;42;0;0;17475;26,55;;S;;;Stockholm, Sweden
-1;1;Lindstrom, Mrs. Carl Johan (Sigrid Posse);female;55;0;0;112377;27,7208;;C;6;;Stockholm, Sweden
-1;1;Lines, Miss. Mary Conover;female;16;0;1;PC 17592;39,4;D28;S;9;;Paris, France
-1;1;Lines, Mrs. Ernest H (Elizabeth Lindsey James);female;51;0;1;PC 17592;39,4;D28;S;9;;Paris, France
-1;0;Long, Mr. Milton Clyde;male;29;0;0;113501;30;D6;S;;126;Springfield, MA
-1;1;Longley, Miss. Gretchen Fiske;female;21;0;0;13502;77,9583;D9;S;10;;Hudson, NY
-1;0;Loring, Mr. Joseph Holland;male;30;0;0;113801;45,5;;S;;;London / New York, NY
-1;1;Lurette, Miss. Elise;female;58;0;0;PC 17569;146,5208;B80;C;;;
-1;1;Madill, Miss. Georgette Alexandra;female;15;0;1;24160;211,3375;B5;S;2;;St Louis, MO
-1;0;Maguire, Mr. John Edward;male;30;0;0;110469;26;C106;S;;;Brockton, MA
-1;1;Maioni, Miss. Roberta;female;16;0;0;110152;86,5;B79;S;8;;
-1;1;Marechal, Mr. Pierre;male;;0;0;11774;29,7;C47;C;7;;Paris, France
-1;0;Marvin, Mr. Daniel Warner;male;19;1;0;113773;53,1;D30;S;;;New York, NY
-1;1;Marvin, Mrs. Daniel Warner (Mary Graham Carmichael Farquarson);female;18;1;0;113773;53,1;D30;S;10;;New York, NY
-1;1;"Mayne, Mlle. Berthe Antonine (""Mrs de Villiers"")";female;24;0;0;PC 17482;49,5042;C90;C;6;;Belgium Montreal, PQ
-1;0;McCaffry, Mr. Thomas Francis;male;46;0;0;13050;75,2417;C6;C;;292;Vancouver, BC
-1;0;McCarthy, Mr. Timothy J;male;54;0;0;17463;51,8625;E46;S;;175;Dorchester, MA
-1;1;McGough, Mr. James Robert;male;36;0;0;PC 17473;26,2875;E25;S;7;;Philadelphia, PA
-1;0;Meyer, Mr. Edgar Joseph;male;28;1;0;PC 17604;82,1708;;C;;;New York, NY
-1;1;Meyer, Mrs. Edgar Joseph (Leila Saks);female;;1;0;PC 17604;82,1708;;C;6;;New York, NY
-1;0;Millet, Mr. Francis Davis;male;65;0;0;13509;26,55;E38;S;;249;East Bridgewater, MA
-1;0;Minahan, Dr. William Edward;male;44;2;0;19928;90;C78;Q;;230;Fond du Lac, WI
-1;1;Minahan, Miss. Daisy E;female;33;1;0;19928;90;C78;Q;14;;Green Bay, WI
-1;1;Minahan, Mrs. William Edward (Lillian E Thorpe);female;37;1;0;19928;90;C78;Q;14;;Fond du Lac, WI
-1;1;Mock, Mr. Philipp Edmund;male;30;1;0;13236;57,75;C78;C;11;;New York, NY
-1;0;Molson, Mr. Harry Markland;male;55;0;0;113787;30,5;C30;S;;;Montreal, PQ
-1;0;Moore, Mr. Clarence Bloomfield;male;47;0;0;113796;42,4;;S;;;Washington, DC
-1;0;Natsch, Mr. Charles H;male;37;0;1;PC 17596;29,7;C118;C;;;Brooklyn, NY
-1;1;Newell, Miss. Madeleine;female;31;1;0;35273;113,275;D36;C;6;;Lexington, MA
-1;1;Newell, Miss. Marjorie;female;23;1;0;35273;113,275;D36;C;6;;Lexington, MA
-1;0;Newell, Mr. Arthur Webster;male;58;0;2;35273;113,275;D48;C;;122;Lexington, MA
-1;1;Newsom, Miss. Helen Monypeny;female;19;0;2;11752;26,2833;D47;S;5;;New York, NY
-1;0;Nicholson, Mr. Arthur Ernest;male;64;0;0;693;26;;S;;263;Isle of Wight, England
-1;1;Oliva y Ocana, Dona. Fermina;female;39;0;0;PC 17758;108,9;C105;C;8;;
-1;1;Omont, Mr. Alfred Fernand;male;;0;0;F.C. 12998;25,7417;;C;7;;Paris, France
-1;1;Ostby, Miss. Helene Ragnhild;female;22;0;1;113509;61,9792;B36;C;5;;Providence, RI
-1;0;Ostby, Mr. Engelhart Cornelius;male;65;0;1;113509;61,9792;B30;C;;234;Providence, RI
-1;0;Ovies y Rodriguez, Mr. Servando;male;28,5;0;0;PC 17562;27,7208;D43;C;;189;?Havana, Cuba
-1;0;Parr, Mr. William Henry Marsh;male;;0;0;112052;0;;S;;;Belfast
-1;0;Partner, Mr. Austen;male;45,5;0;0;113043;28,5;C124;S;;166;Surbiton Hill, Surrey
-1;0;Payne, Mr. Vivian Ponsonby;male;23;0;0;12749;93,5;B24;S;;;Montreal, PQ
-1;0;Pears, Mr. Thomas Clinton;male;29;1;0;113776;66,6;C2;S;;;Isleworth, England
-1;1;Pears, Mrs. Thomas (Edith Wearne);female;22;1;0;113776;66,6;C2;S;8;;Isleworth, England
-1;0;Penasco y Castellana, Mr. Victor de Satode;male;18;1;0;PC 17758;108,9;C65;C;;;Madrid, Spain
-1;1;Penasco y Castellana, Mrs. Victor de Satode (Maria Josefa Perez de Soto y Vallejo);female;17;1;0;PC 17758;108,9;C65;C;8;;Madrid, Spain
-1;1;Perreault, Miss. Anne;female;30;0;0;12749;93,5;B73;S;3;;
-1;1;Peuchen, Major. Arthur Godfrey;male;52;0;0;113786;30,5;C104;S;6;;Toronto, ON
-1;0;Porter, Mr. Walter Chamberlain;male;47;0;0;110465;52;C110;S;;207;Worcester, MA
-1;1;Potter, Mrs. Thomas Jr (Lily Alexenia Wilson);female;56;0;1;11767;83,1583;C50;C;7;;Mt Airy, Philadelphia, PA
-1;0;Reuchlin, Jonkheer. John George;male;38;0;0;19972;0;;S;;;Rotterdam, Netherlands
-1;1;Rheims, Mr. George Alexander Lucien;male;;0;0;PC 17607;39,6;;S;A;;Paris / New York, NY
-1;0;Ringhini, Mr. Sante;male;22;0;0;PC 17760;135,6333;;C;;232;
-1;0;Robbins, Mr. Victor;male;;0;0;PC 17757;227,525;;C;;;
-1;1;Robert, Mrs. Edward Scott (Elisabeth Walton McMillan);female;43;0;1;24160;211,3375;B3;S;2;;St Louis, MO
-1;0;Roebling, Mr. Washington Augustus II;male;31;0;0;PC 17590;50,4958;A24;S;;;Trenton, NJ
-1;1;"Romaine, Mr. Charles Hallace (""Mr C Rolmane"")";male;45;0;0;111428;26,55;;S;9;;New York, NY
-1;0;Rood, Mr. Hugh Roscoe;male;;0;0;113767;50;A32;S;;;Seattle, WA
-1;1;Rosenbaum, Miss. Edith Louise;female;33;0;0;PC 17613;27,7208;A11;C;11;;Paris, France
-1;0;"Rosenshine, Mr. George (""Mr George Thorne"")";male;46;0;0;PC 17585;79,2;;C;;16;New York, NY
-1;0;Ross, Mr. John Hugo;male;36;0;0;13049;40,125;A10;C;;;Winnipeg, MB
-1;1;Rothes, the Countess. of (Lucy Noel Martha Dyer-Edwards);female;33;0;0;110152;86,5;B77;S;8;;London Vancouver, BC
-1;0;Rothschild, Mr. Martin;male;55;1;0;PC 17603;59,4;;C;;;New York, NY
-1;1;Rothschild, Mrs. Martin (Elizabeth L. Barrett);female;54;1;0;PC 17603;59,4;;C;6;;New York, NY
-1;0;Rowe, Mr. Alfred G;male;33;0;0;113790;26,55;;S;;109;London
-1;1;Ryerson, Master. John Borie;male;13;2;2;PC 17608;262,375;B57 B59 B63 B66;C;4;;Haverford, PA / Cooperstown, NY
-1;1;Ryerson, Miss. Emily Borie;female;18;2;2;PC 17608;262,375;B57 B59 B63 B66;C;4;;Haverford, PA / Cooperstown, NY
-1;1;"Ryerson, Miss. Susan Parker ""Suzette""";female;21;2;2;PC 17608;262,375;B57 B59 B63 B66;C;4;;Haverford, PA / Cooperstown, NY
-1;0;Ryerson, Mr. Arthur Larned;male;61;1;3;PC 17608;262,375;B57 B59 B63 B66;C;;;Haverford, PA / Cooperstown, NY
-1;1;Ryerson, Mrs. Arthur Larned (Emily Maria Borie);female;48;1;3;PC 17608;262,375;B57 B59 B63 B66;C;4;;Haverford, PA / Cooperstown, NY
-1;1;Saalfeld, Mr. Adolphe;male;;0;0;19988;30,5;C106;S;3;;Manchester, England
-1;1;Sagesser, Mlle. Emma;female;24;0;0;PC 17477;69,3;B35;C;9;;
-1;1;Salomon, Mr. Abraham L;male;;0;0;111163;26;;S;1;;New York, NY
-1;1;Schabert, Mrs. Paul (Emma Mock);female;35;1;0;13236;57,75;C28;C;11;;New York, NY
-1;1;Serepeca, Miss. Augusta;female;30;0;0;113798;31;;C;4;;
-1;1;Seward, Mr. Frederic Kimber;male;34;0;0;113794;26,55;;S;7;;New York, NY
-1;1;Shutes, Miss. Elizabeth W;female;40;0;0;PC 17582;153,4625;C125;S;3;;New York, NY / Greenwich CT
-1;1;Silverthorne, Mr. Spencer Victor;male;35;0;0;PC 17475;26,2875;E24;S;5;;St Louis, MO
-1;0;Silvey, Mr. William Baird;male;50;1;0;13507;55,9;E44;S;;;Duluth, MN
-1;1;Silvey, Mrs. William Baird (Alice Munger);female;39;1;0;13507;55,9;E44;S;11;;Duluth, MN
-1;1;Simonius-Blumer, Col. Oberst Alfons;male;56;0;0;13213;35,5;A26;C;3;;Basel, Switzerland
-1;1;Sloper, Mr. William Thompson;male;28;0;0;113788;35,5;A6;S;7;;New Britain, CT
-1;0;Smart, Mr. John Montgomery;male;56;0;0;113792;26,55;;S;;;New York, NY
-1;0;Smith, Mr. James Clinch;male;56;0;0;17764;30,6958;A7;C;;;St James, Long Island, NY
-1;0;Smith, Mr. Lucien Philip;male;24;1;0;13695;60;C31;S;;;Huntington, WV
-1;0;Smith, Mr. Richard William;male;;0;0;113056;26;A19;S;;;Streatham, Surrey
-1;1;Smith, Mrs. Lucien Philip (Mary Eloise Hughes);female;18;1;0;13695;60;C31;S;6;;Huntington, WV
-1;1;Snyder, Mr. John Pillsbury;male;24;1;0;21228;82,2667;B45;S;7;;Minneapolis, MN
-1;1;Snyder, Mrs. John Pillsbury (Nelle Stevenson);female;23;1;0;21228;82,2667;B45;S;7;;Minneapolis, MN
-1;1;Spedden, Master. Robert Douglas;male;6;0;2;16966;134,5;E34;C;3;;Tuxedo Park, NY
-1;1;Spedden, Mr. Frederic Oakley;male;45;1;1;16966;134,5;E34;C;3;;Tuxedo Park, NY
-1;1;Spedden, Mrs. Frederic Oakley (Margaretta Corning Stone);female;40;1;1;16966;134,5;E34;C;3;;Tuxedo Park, NY
-1;0;Spencer, Mr. William Augustus;male;57;1;0;PC 17569;146,5208;B78;C;;;Paris, France
-1;1;Spencer, Mrs. William Augustus (Marie Eugenie);female;;1;0;PC 17569;146,5208;B78;C;6;;Paris, France
-1;1;Stahelin-Maeglin, Dr. Max;male;32;0;0;13214;30,5;B50;C;3;;Basel, Switzerland
-1;0;Stead, Mr. William Thomas;male;62;0;0;113514;26,55;C87;S;;;Wimbledon Park, London / Hayling Island, Hants
-1;1;Stengel, Mr. Charles Emil Henry;male;54;1;0;11778;55,4417;C116;C;1;;Newark, NJ
-1;1;Stengel, Mrs. Charles Emil Henry (Annie May Morris);female;43;1;0;11778;55,4417;C116;C;5;;Newark, NJ
-1;1;Stephenson, Mrs. Walter Bertram (Martha Eustis);female;52;1;0;36947;78,2667;D20;C;4;;Haverford, PA
-1;0;Stewart, Mr. Albert A;male;;0;0;PC 17605;27,7208;;C;;;Gallipolis, Ohio / ? Paris / New York
-1;1;Stone, Mrs. George Nelson (Martha Evelyn);female;62;0;0;113572;80;B28;;6;;Cincinatti, OH
-1;0;Straus, Mr. Isidor;male;67;1;0;PC 17483;221,7792;C55 C57;S;;96;New York, NY
-1;0;Straus, Mrs. Isidor (Rosalie Ida Blun);female;63;1;0;PC 17483;221,7792;C55 C57;S;;;New York, NY
-1;0;Sutton, Mr. Frederick;male;61;0;0;36963;32,3208;D50;S;;46;Haddenfield, NJ
-1;1;Swift, Mrs. Frederick Joel (Margaret Welles Barron);female;48;0;0;17466;25,9292;D17;S;8;;Brooklyn, NY
-1;1;Taussig, Miss. Ruth;female;18;0;2;110413;79,65;E68;S;8;;New York, NY
-1;0;Taussig, Mr. Emil;male;52;1;1;110413;79,65;E67;S;;;New York, NY
-1;1;Taussig, Mrs. Emil (Tillie Mandelbaum);female;39;1;1;110413;79,65;E67;S;8;;New York, NY
-1;1;Taylor, Mr. Elmer Zebley;male;48;1;0;19996;52;C126;S;5 7;;London / East Orange, NJ
-1;1;Taylor, Mrs. Elmer Zebley (Juliet Cummins Wright);female;;1;0;19996;52;C126;S;5 7;;London / East Orange, NJ
-1;0;Thayer, Mr. John Borland;male;49;1;1;17421;110,8833;C68;C;;;Haverford, PA
-1;1;Thayer, Mr. John Borland Jr;male;17;0;2;17421;110,8833;C70;C;B;;Haverford, PA
-1;1;Thayer, Mrs. John Borland (Marian Longstreth Morris);female;39;1;1;17421;110,8833;C68;C;4;;Haverford, PA
-1;1;Thorne, Mrs. Gertrude Maybelle;female;;0;0;PC 17585;79,2;;C;D;;New York, NY
-1;1;Tucker, Mr. Gilbert Milligan Jr;male;31;0;0;2543;28,5375;C53;C;7;;Albany, NY
-1;0;Uruchurtu, Don. Manuel E;male;40;0;0;PC 17601;27,7208;;C;;;Mexico City, Mexico
-1;0;Van der hoef, Mr. Wyckoff;male;61;0;0;111240;33,5;B19;S;;245;Brooklyn, NY
-1;0;Walker, Mr. William Anderson;male;47;0;0;36967;34,0208;D46;S;;;East Orange, NJ
-1;1;Ward, Miss. Anna;female;35;0;0;PC 17755;512,3292;;C;3;;
-1;0;Warren, Mr. Frank Manley;male;64;1;0;110813;75,25;D37;C;;;Portland, OR
-1;1;Warren, Mrs. Frank Manley (Anna Sophia Atkinson);female;60;1;0;110813;75,25;D37;C;5;;Portland, OR
-1;0;Weir, Col. John;male;60;0;0;113800;26,55;;S;;;England Salt Lake City, Utah
-1;0;White, Mr. Percival Wayland;male;54;0;1;35281;77,2875;D26;S;;;Brunswick, ME
-1;0;White, Mr. Richard Frasar;male;21;0;1;35281;77,2875;D26;S;;169;Brunswick, ME
-1;1;White, Mrs. John Stuart (Ella Holmes);female;55;0;0;PC 17760;135,6333;C32;C;8;;New York, NY / Briarcliff Manor NY
-1;1;Wick, Miss. Mary Natalie;female;31;0;2;36928;164,8667;C7;S;8;;Youngstown, OH
-1;0;Wick, Mr. George Dennick;male;57;1;1;36928;164,8667;;S;;;Youngstown, OH
-1;1;Wick, Mrs. George Dennick (Mary Hitchcock);female;45;1;1;36928;164,8667;;S;8;;Youngstown, OH
-1;0;Widener, Mr. George Dunton;male;50;1;1;113503;211,5;C80;C;;;Elkins Park, PA
-1;0;Widener, Mr. Harry Elkins;male;27;0;2;113503;211,5;C82;C;;;Elkins Park, PA
-1;1;Widener, Mrs. George Dunton (Eleanor Elkins);female;50;1;1;113503;211,5;C80;C;4;;Elkins Park, PA
-1;1;Willard, Miss. Constance;female;21;0;0;113795;26,55;;S;8 10;;Duluth, MN
-1;0;Williams, Mr. Charles Duane;male;51;0;1;PC 17597;61,3792;;C;;;Geneva, Switzerland / Radnor, PA
-1;1;Williams, Mr. Richard Norris II;male;21;0;1;PC 17597;61,3792;;C;A;;Geneva, Switzerland / Radnor, PA
-1;0;Williams-Lambert, Mr. Fletcher Fellows;male;;0;0;113510;35;C128;S;;;London, England
-1;1;Wilson, Miss. Helen Alice;female;31;0;0;16966;134,5;E39 E41;C;3;;
-1;1;Woolner, Mr. Hugh;male;;0;0;19947;35,5;C52;S;D;;London, England
-1;0;Wright, Mr. George;male;62;0;0;113807;26,55;;S;;;Halifax, NS
-1;1;Young, Miss. Marie Grice;female;36;0;0;PC 17760;135,6333;C32;C;8;;New York, NY / Washington, DC
-2;0;Abelson, Mr. Samuel;male;30;1;0;P/PP 3381;24;;C;;;Russia New York, NY
-2;1;Abelson, Mrs. Samuel (Hannah Wizosky);female;28;1;0;P/PP 3381;24;;C;10;;Russia New York, NY
-2;0;Aldworth, Mr. Charles Augustus;male;30;0;0;248744;13;;S;;;Bryn Mawr, PA, USA
-2;0;Andrew, Mr. Edgardo Samuel;male;18;0;0;231945;11,5;;S;;;Buenos Aires, Argentina / New Jersey, NJ
-2;0;Andrew, Mr. Frank Thomas;male;25;0;0;C.A. 34050;10,5;;S;;;Cornwall, England Houghton, MI
-2;0;Angle, Mr. William A;male;34;1;0;226875;26;;S;;;Warwick, England
-2;1;"Angle, Mrs. William A (Florence ""Mary"" Agnes Hughes)";female;36;1;0;226875;26;;S;11;;Warwick, England
-2;0;Ashby, Mr. John;male;57;0;0;244346;13;;S;;;West Hoboken, NJ
-2;0;Bailey, Mr. Percy Andrew;male;18;0;0;29108;11,5;;S;;;Penzance, Cornwall / Akron, OH
-2;0;Baimbrigge, Mr. Charles Robert;male;23;0;0;C.A. 31030;10,5;;S;;;Guernsey
-2;1;Ball, Mrs. (Ada E Hall);female;36;0;0;28551;13;D;S;10;;Bristol, Avon / Jacksonville, FL
-2;0;Banfield, Mr. Frederick James;male;28;0;0;C.A./SOTON 34068;10,5;;S;;;Plymouth, Dorset / Houghton, MI
-2;0;Bateman, Rev. Robert James;male;51;0;0;S.O.P. 1166;12,525;;S;;174;Jacksonville, FL
-2;1;Beane, Mr. Edward;male;32;1;0;2908;26;;S;13;;Norwich / New York, NY
-2;1;Beane, Mrs. Edward (Ethel Clarke);female;19;1;0;2908;26;;S;13;;Norwich / New York, NY
-2;0;Beauchamp, Mr. Henry James;male;28;0;0;244358;26;;S;;;England
-2;1;Becker, Master. Richard F;male;1;2;1;230136;39;F4;S;11;;Guntur, India / Benton Harbour, MI
-2;1;Becker, Miss. Marion Louise;female;4;2;1;230136;39;F4;S;11;;Guntur, India / Benton Harbour, MI
-2;1;Becker, Miss. Ruth Elizabeth;female;12;2;1;230136;39;F4;S;13;;Guntur, India / Benton Harbour, MI
-2;1;Becker, Mrs. Allen Oliver (Nellie E Baumgardner);female;36;0;3;230136;39;F4;S;11;;Guntur, India / Benton Harbour, MI
-2;1;Beesley, Mr. Lawrence;male;34;0;0;248698;13;D56;S;13;;London
-2;1;Bentham, Miss. Lilian W;female;19;0;0;28404;13;;S;12;;Rochester, NY
-2;0;Berriman, Mr. William John;male;23;0;0;28425;13;;S;;;St Ives, Cornwall / Calumet, MI
-2;0;Botsford, Mr. William Hull;male;26;0;0;237670;13;;S;;;Elmira, NY / Orange, NJ
-2;0;Bowenur, Mr. Solomon;male;42;0;0;211535;13;;S;;;London
-2;0;Bracken, Mr. James H;male;27;0;0;220367;13;;S;;;Lake Arthur, Chavez County, NM
-2;1;"Brown, Miss. Amelia ""Mildred""";female;24;0;0;248733;13;F33;S;11;;London / Montreal, PQ
-2;1;Brown, Miss. Edith Eileen;female;15;0;2;29750;39;;S;14;;Cape Town, South Africa / Seattle, WA
-2;0;Brown, Mr. Thomas William Solomon;male;60;1;1;29750;39;;S;;;Cape Town, South Africa / Seattle, WA
-2;1;Brown, Mrs. Thomas William Solomon (Elizabeth Catherine Ford);female;40;1;1;29750;39;;S;14;;Cape Town, South Africa / Seattle, WA
-2;1;Bryhl, Miss. Dagmar Jenny Ingeborg ;female;20;1;0;236853;26;;S;12;;Skara, Sweden / Rockford, IL
-2;0;Bryhl, Mr. Kurt Arnold Gottfrid;male;25;1;0;236853;26;;S;;;Skara, Sweden / Rockford, IL
-2;1;Buss, Miss. Kate;female;36;0;0;27849;13;;S;9;;Sittingbourne, England / San Diego, CA
-2;0;Butler, Mr. Reginald Fenton;male;25;0;0;234686;13;;S;;97;Southsea, Hants
-2;0;Byles, Rev. Thomas Roussel Davids;male;42;0;0;244310;13;;S;;;London
-2;1;Bystrom, Mrs. (Karolina);female;42;0;0;236852;13;;S;;;New York, NY
-2;1;Caldwell, Master. Alden Gates;male;0,8333;0;2;248738;29;;S;13;;Bangkok, Thailand / Roseville, IL
-2;1;Caldwell, Mr. Albert Francis;male;26;1;1;248738;29;;S;13;;Bangkok, Thailand / Roseville, IL
-2;1;Caldwell, Mrs. Albert Francis (Sylvia Mae Harbaugh);female;22;1;1;248738;29;;S;13;;Bangkok, Thailand / Roseville, IL
-2;1;Cameron, Miss. Clear Annie;female;35;0;0;F.C.C. 13528;21;;S;14;;Mamaroneck, NY
-2;0;Campbell, Mr. William;male;;0;0;239853;0;;S;;;Belfast
-2;0;Carbines, Mr. William;male;19;0;0;28424;13;;S;;18;St Ives, Cornwall / Calumet, MI
-2;0;Carter, Mrs. Ernest Courtenay (Lilian Hughes);female;44;1;0;244252;26;;S;;;London
-2;0;Carter, Rev. Ernest Courtenay;male;54;1;0;244252;26;;S;;;London
-2;0;Chapman, Mr. Charles Henry;male;52;0;0;248731;13,5;;S;;130;Bronx, NY
-2;0;Chapman, Mr. John Henry;male;37;1;0;SC/AH 29037;26;;S;;17;Cornwall / Spokane, WA
-2;0;Chapman, Mrs. John Henry (Sara Elizabeth Lawry);female;29;1;0;SC/AH 29037;26;;S;;;Cornwall / Spokane, WA
-2;1;Christy, Miss. Julie Rachel;female;25;1;1;237789;30;;S;12;;London
-2;1;Christy, Mrs. (Alice Frances);female;45;0;2;237789;30;;S;12;;London
-2;0;Clarke, Mr. Charles Valentine;male;29;1;0;2003;26;;S;;;England / San Francisco, CA
-2;1;Clarke, Mrs. Charles V (Ada Maria Winfield);female;28;1;0;2003;26;;S;14;;England / San Francisco, CA
-2;0;Coleridge, Mr. Reginald Charles;male;29;0;0;W./C. 14263;10,5;;S;;;Hartford, Huntingdonshire
-2;0;Collander, Mr. Erik Gustaf;male;28;0;0;248740;13;;S;;;Helsinki, Finland Ashtabula, Ohio
-2;1;Collett, Mr. Sidney C Stuart;male;24;0;0;28034;10,5;;S;9;;London / Fort Byron, NY
-2;1;"Collyer, Miss. Marjorie ""Lottie""";female;8;0;2;C.A. 31921;26,25;;S;14;;Bishopstoke, Hants / Fayette Valley, ID
-2;0;Collyer, Mr. Harvey;male;31;1;1;C.A. 31921;26,25;;S;;;Bishopstoke, Hants / Fayette Valley, ID
-2;1;Collyer, Mrs. Harvey (Charlotte Annie Tate);female;31;1;1;C.A. 31921;26,25;;S;14;;Bishopstoke, Hants / Fayette Valley, ID
-2;1;Cook, Mrs. (Selena Rogers);female;22;0;0;W./C. 14266;10,5;F33;S;14;;Pennsylvania
-2;0;Corbett, Mrs. Walter H (Irene Colvin);female;30;0;0;237249;13;;S;;;Provo, UT
-2;0;Corey, Mrs. Percy C (Mary Phyllis Elizabeth Miller);female;;0;0;F.C.C. 13534;21;;S;;;Upper Burma, India Pittsburgh, PA
-2;0;"Cotterill, Mr. Henry ""Harry""";male;21;0;0;29107;11,5;;S;;;Penzance, Cornwall / Akron, OH
-2;0;Cunningham, Mr. Alfred Fleming;male;;0;0;239853;0;;S;;;Belfast
-2;1;Davies, Master. John Morgan Jr;male;8;1;1;C.A. 33112;36,75;;S;14;;St Ives, Cornwall / Hancock, MI
-2;0;Davies, Mr. Charles Henry;male;18;0;0;S.O.C. 14879;73,5;;S;;;Lyndhurst, England
-2;1;Davies, Mrs. John Morgan (Elizabeth Agnes Mary White) ;female;48;0;2;C.A. 33112;36,75;;S;14;;St Ives, Cornwall / Hancock, MI
-2;1;Davis, Miss. Mary;female;28;0;0;237668;13;;S;13;;London / Staten Island, NY
-2;0;de Brito, Mr. Jose Joaquim;male;32;0;0;244360;13;;S;;;Portugal / Sau Paulo, Brazil
-2;0;Deacon, Mr. Percy William;male;17;0;0;S.O.C. 14879;73,5;;S;;;
-2;0;del Carlo, Mr. Sebastiano;male;29;1;0;SC/PARIS 2167;27,7208;;C;;295;Lucca, Italy / California
-2;1;del Carlo, Mrs. Sebastiano (Argenia Genovesi);female;24;1;0;SC/PARIS 2167;27,7208;;C;12;;Lucca, Italy / California
-2;0;Denbury, Mr. Herbert;male;25;0;0;C.A. 31029;31,5;;S;;;Guernsey / Elizabeth, NJ
-2;0;Dibden, Mr. William;male;18;0;0;S.O.C. 14879;73,5;;S;;;New Forest, England
-2;1;Doling, Miss. Elsie;female;18;0;1;231919;23;;S;;;Southampton
-2;1;Doling, Mrs. John T (Ada Julia Bone);female;34;0;1;231919;23;;S;;;Southampton
-2;0;Downton, Mr. William James;male;54;0;0;28403;26;;S;;;Holley, NY
-2;1;Drew, Master. Marshall Brines;male;8;0;2;28220;32,5;;S;10;;Greenport, NY
-2;0;Drew, Mr. James Vivian;male;42;1;1;28220;32,5;;S;;;Greenport, NY
-2;1;Drew, Mrs. James Vivian (Lulu Thorne Christian);female;34;1;1;28220;32,5;;S;10;;Greenport, NY
-2;1;Duran y More, Miss. Asuncion;female;27;1;0;SC/PARIS 2149;13,8583;;C;12;;Barcelona, Spain / Havana, Cuba
-2;1;Duran y More, Miss. Florentina;female;30;1;0;SC/PARIS 2148;13,8583;;C;12;;Barcelona, Spain / Havana, Cuba
-2;0;Eitemiller, Mr. George Floyd;male;23;0;0;29751;13;;S;;;England / Detroit, MI
-2;0;Enander, Mr. Ingvar;male;21;0;0;236854;13;;S;;;Goteborg, Sweden / Rockford, IL
-2;0;Fahlstrom, Mr. Arne Jonas;male;18;0;0;236171;13;;S;;;Oslo, Norway Bayonne, NJ
-2;0;Faunthorpe, Mr. Harry;male;40;1;0;2926;26;;S;;286;England / Philadelphia, PA
-2;1;Faunthorpe, Mrs. Lizzie (Elizabeth Anne Wilkinson);female;29;1;0;2926;26;;S;16;;
-2;0;Fillbrook, Mr. Joseph Charles;male;18;0;0;C.A. 15185;10,5;;S;;;Cornwall / Houghton, MI
-2;0;Fox, Mr. Stanley Hubert;male;36;0;0;229236;13;;S;;236;Rochester, NY
-2;0;"Frost, Mr. Anthony Wood ""Archie""";male;;0;0;239854;0;;S;;;Belfast
-2;0;Funk, Miss. Annie Clemmer;female;38;0;0;237671;13;;S;;;Janjgir, India / Pennsylvania
-2;0;Fynney, Mr. Joseph J;male;35;0;0;239865;26;;S;;322;Liverpool / Montreal, PQ
-2;0;Gale, Mr. Harry;male;38;1;0;28664;21;;S;;;Cornwall / Clear Creek, CO
-2;0;Gale, Mr. Shadrach;male;34;1;0;28664;21;;S;;;Cornwall / Clear Creek, CO
-2;1;Garside, Miss. Ethel;female;34;0;0;243880;13;;S;12;;Brooklyn, NY
-2;0;Gaskell, Mr. Alfred;male;16;0;0;239865;26;;S;;;Liverpool / Montreal, PQ
-2;0;Gavey, Mr. Lawrence;male;26;0;0;31028;10,5;;S;;;Guernsey / Elizabeth, NJ
-2;0;Gilbert, Mr. William;male;47;0;0;C.A. 30769;10,5;;S;;;Cornwall
-2;0;Giles, Mr. Edgar;male;21;1;0;28133;11,5;;S;;;Cornwall / Camden, NJ
-2;0;Giles, Mr. Frederick Edward;male;21;1;0;28134;11,5;;S;;;Cornwall / Camden, NJ
-2;0;Giles, Mr. Ralph;male;24;0;0;248726;13,5;;S;;297;West Kensington, London
-2;0;Gill, Mr. John William;male;24;0;0;233866;13;;S;;155;Clevedon, England
-2;0;Gillespie, Mr. William Henry;male;34;0;0;12233;13;;S;;;Vancouver, BC
-2;0;Givard, Mr. Hans Kristensen;male;30;0;0;250646;13;;S;;305;
-2;0;Greenberg, Mr. Samuel;male;52;0;0;250647;13;;S;;19;Bronx, NY
-2;0;Hale, Mr. Reginald;male;30;0;0;250653;13;;S;;75;Auburn, NY
-2;1;Hamalainen, Master. Viljo;male;0,6667;1;1;250649;14,5;;S;4;;Detroit, MI
-2;1;Hamalainen, Mrs. William (Anna);female;24;0;2;250649;14,5;;S;4;;Detroit, MI
-2;0;Harbeck, Mr. William H;male;44;0;0;248746;13;;S;;35;Seattle, WA / Toledo, OH
-2;1;"Harper, Miss. Annie Jessie ""Nina""";female;6;0;1;248727;33;;S;11;;Denmark Hill, Surrey / Chicago
-2;0;Harper, Rev. John;male;28;0;1;248727;33;;S;;;Denmark Hill, Surrey / Chicago
-2;1;Harris, Mr. George;male;62;0;0;S.W./PP 752;10,5;;S;15;;London
-2;0;Harris, Mr. Walter;male;30;0;0;W/C 14208;10,5;;S;;;Walthamstow, England
-2;1;Hart, Miss. Eva Miriam;female;7;0;2;F.C.C. 13529;26,25;;S;14;;Ilford, Essex / Winnipeg, MB
-2;0;Hart, Mr. Benjamin;male;43;1;1;F.C.C. 13529;26,25;;S;;;Ilford, Essex / Winnipeg, MB
-2;1;Hart, Mrs. Benjamin (Esther Ada Bloomfield);female;45;1;1;F.C.C. 13529;26,25;;S;14;;Ilford, Essex / Winnipeg, MB
-2;1;Herman, Miss. Alice;female;24;1;2;220845;65;;S;9;;Somerset / Bernardsville, NJ
-2;1;Herman, Miss. Kate;female;24;1;2;220845;65;;S;9;;Somerset / Bernardsville, NJ
-2;0;Herman, Mr. Samuel;male;49;1;2;220845;65;;S;;;Somerset / Bernardsville, NJ
-2;1;Herman, Mrs. Samuel (Jane Laver);female;48;1;2;220845;65;;S;9;;Somerset / Bernardsville, NJ
-2;1;Hewlett, Mrs. (Mary D Kingcome) ;female;55;0;0;248706;16;;S;13;;India / Rapid City, SD
-2;0;Hickman, Mr. Leonard Mark;male;24;2;0;S.O.C. 14879;73,5;;S;;;West Hampstead, London / Neepawa, MB
-2;0;Hickman, Mr. Lewis;male;32;2;0;S.O.C. 14879;73,5;;S;;256;West Hampstead, London / Neepawa, MB
-2;0;Hickman, Mr. Stanley George;male;21;2;0;S.O.C. 14879;73,5;;S;;;West Hampstead, London / Neepawa, MB
-2;0;Hiltunen, Miss. Marta;female;18;1;1;250650;13;;S;;;Kontiolahti, Finland / Detroit, MI
-2;1;"Hocking, Miss. Ellen ""Nellie""";female;20;2;1;29105;23;;S;4;;Cornwall / Akron, OH
-2;0;Hocking, Mr. Richard George;male;23;2;1;29104;11,5;;S;;;Cornwall / Akron, OH
-2;0;Hocking, Mr. Samuel James Metcalfe;male;36;0;0;242963;13;;S;;;Devonport, England
-2;1;Hocking, Mrs. Elizabeth (Eliza Needs);female;54;1;3;29105;23;;S;4;;Cornwall / Akron, OH
-2;0;Hodges, Mr. Henry Price;male;50;0;0;250643;13;;S;;149;Southampton
-2;0;Hold, Mr. Stephen;male;44;1;0;26707;26;;S;;;England / Sacramento, CA
-2;1;Hold, Mrs. Stephen (Annie Margaret Hill);female;29;1;0;26707;26;;S;10;;England / Sacramento, CA
-2;0;Hood, Mr. Ambrose Jr;male;21;0;0;S.O.C. 14879;73,5;;S;;;New Forest, England
-2;1;Hosono, Mr. Masabumi;male;42;0;0;237798;13;;S;10;;Tokyo, Japan
-2;0;Howard, Mr. Benjamin;male;63;1;0;24065;26;;S;;;Swindon, England
-2;0;Howard, Mrs. Benjamin (Ellen Truelove Arman);female;60;1;0;24065;26;;S;;;Swindon, England
-2;0;Hunt, Mr. George Henry;male;33;0;0;SCO/W 1585;12,275;;S;;;Philadelphia, PA
-2;1;Ilett, Miss. Bertha;female;17;0;0;SO/C 14885;10,5;;S;;;Guernsey
-2;0;Jacobsohn, Mr. Sidney Samuel;male;42;1;0;243847;27;;S;;;London
-2;1;Jacobsohn, Mrs. Sidney Samuel (Amy Frances Christy);female;24;2;1;243847;27;;S;12;;London
-2;0;Jarvis, Mr. John Denzil;male;47;0;0;237565;15;;S;;;North Evington, England
-2;0;Jefferys, Mr. Clifford Thomas;male;24;2;0;C.A. 31029;31,5;;S;;;Guernsey / Elizabeth, NJ
-2;0;Jefferys, Mr. Ernest Wilfred;male;22;2;0;C.A. 31029;31,5;;S;;;Guernsey / Elizabeth, NJ
-2;0;Jenkin, Mr. Stephen Curnow;male;32;0;0;C.A. 33111;10,5;;S;;;St Ives, Cornwall / Houghton, MI
-2;1;Jerwan, Mrs. Amin S (Marie Marthe Thuillard);female;23;0;0;SC/AH Basle 541;13,7917;D;C;11;;New York, NY
-2;0;Kantor, Mr. Sinai;male;34;1;0;244367;26;;S;;283;Moscow / Bronx, NY
-2;1;Kantor, Mrs. Sinai (Miriam Sternin);female;24;1;0;244367;26;;S;12;;Moscow / Bronx, NY
-2;0;Karnes, Mrs. J Frank (Claire Bennett);female;22;0;0;F.C.C. 13534;21;;S;;;India / Pittsburgh, PA
-2;1;Keane, Miss. Nora A;female;;0;0;226593;12,35;E101;Q;10;;Harrisburg, PA
-2;0;Keane, Mr. Daniel;male;35;0;0;233734;12,35;;Q;;;
-2;1;"Kelly, Mrs. Florence ""Fannie""";female;45;0;0;223596;13,5;;S;9;;London / New York, NY
-2;0;Kirkland, Rev. Charles Leonard;male;57;0;0;219533;12,35;;Q;;;Glasgow / Bangor, ME
-2;0;Knight, Mr. Robert J;male;;0;0;239855;0;;S;;;Belfast
-2;0;Kvillner, Mr. Johan Henrik Johannesson;male;31;0;0;C.A. 18723;10,5;;S;;165;Sweden / Arlington, NJ
-2;0;Lahtinen, Mrs. William (Anna Sylfven);female;26;1;1;250651;26;;S;;;Minneapolis, MN
-2;0;Lahtinen, Rev. William;male;30;1;1;250651;26;;S;;;Minneapolis, MN
-2;0;Lamb, Mr. John Joseph;male;;0;0;240261;10,7083;;Q;;;
-2;1;Laroche, Miss. Louise;female;1;1;2;SC/Paris 2123;41,5792;;C;14;;Paris / Haiti
-2;1;Laroche, Miss. Simonne Marie Anne Andree;female;3;1;2;SC/Paris 2123;41,5792;;C;14;;Paris / Haiti
-2;0;Laroche, Mr. Joseph Philippe Lemercier;male;25;1;2;SC/Paris 2123;41,5792;;C;;;Paris / Haiti
-2;1;Laroche, Mrs. Joseph (Juliette Marie Louise Lafargue);female;22;1;2;SC/Paris 2123;41,5792;;C;14;;Paris / Haiti
-2;1;Lehmann, Miss. Bertha;female;17;0;0;SC 1748;12;;C;12;;Berne, Switzerland / Central City, IA
-2;1;Leitch, Miss. Jessie Wills;female;;0;0;248727;33;;S;11;;London / Chicago, IL
-2;1;Lemore, Mrs. (Amelia Milley);female;34;0;0;C.A. 34260;10,5;F33;S;14;;Chicago, IL
-2;0;Levy, Mr. Rene Jacques;male;36;0;0;SC/Paris 2163;12,875;D;C;;;Montreal, PQ
-2;0;Leyson, Mr. Robert William Norman;male;24;0;0;C.A. 29566;10,5;;S;;108;
-2;0;Lingane, Mr. John;male;61;0;0;235509;12,35;;Q;;;
-2;0;Louch, Mr. Charles Alexander;male;50;1;0;SC/AH 3085;26;;S;;121;Weston-Super-Mare, Somerset
-2;1;Louch, Mrs. Charles Alexander (Alice Adelaide Slow);female;42;1;0;SC/AH 3085;26;;S;;;Weston-Super-Mare, Somerset
-2;0;Mack, Mrs. (Mary);female;57;0;0;S.O./P.P. 3;10,5;E77;S;;52;Southampton / New York, NY
-2;0;Malachard, Mr. Noel;male;;0;0;237735;15,0458;D;C;;;Paris
-2;1;Mallet, Master. Andre;male;1;0;2;S.C./PARIS 2079;37,0042;;C;10;;Paris / Montreal, PQ
-2;0;Mallet, Mr. Albert;male;31;1;1;S.C./PARIS 2079;37,0042;;C;;;Paris / Montreal, PQ
-2;1;Mallet, Mrs. Albert (Antoinette Magnin);female;24;1;1;S.C./PARIS 2079;37,0042;;C;10;;Paris / Montreal, PQ
-2;0;Mangiavacchi, Mr. Serafino Emilio;male;;0;0;SC/A.3 2861;15,5792;;C;;;New York, NY
-2;0;Matthews, Mr. William John;male;30;0;0;28228;13;;S;;;St Austall, Cornwall
-2;0;Maybery, Mr. Frank Hubert;male;40;0;0;239059;16;;S;;;Weston-Super-Mare / Moose Jaw, SK
-2;0;McCrae, Mr. Arthur Gordon;male;32;0;0;237216;13,5;;S;;209;Sydney, Australia
-2;0;McCrie, Mr. James Matthew;male;30;0;0;233478;13;;S;;;Sarnia, ON
-2;0;McKane, Mr. Peter David;male;46;0;0;28403;26;;S;;;Rochester, NY
-2;1;Mellinger, Miss. Madeleine Violet;female;13;0;1;250644;19,5;;S;14;;England / Bennington, VT
-2;1;Mellinger, Mrs. (Elizabeth Anne Maidment);female;41;0;1;250644;19,5;;S;14;;England / Bennington, VT
-2;1;Mellors, Mr. William John;male;19;0;0;SW/PP 751;10,5;;S;B;;Chelsea, London
-2;0;Meyer, Mr. August;male;39;0;0;248723;13;;S;;;Harrow-on-the-Hill, Middlesex
-2;0;Milling, Mr. Jacob Christian;male;48;0;0;234360;13;;S;;271;Copenhagen, Denmark
-2;0;Mitchell, Mr. Henry Michael;male;70;0;0;C.A. 24580;10,5;;S;;;Guernsey / Montclair, NJ and/or Toledo, Ohio
-2;0;Montvila, Rev. Juozas;male;27;0;0;211536;13;;S;;;Worcester, MA
-2;0;Moraweck, Dr. Ernest;male;54;0;0;29011;14;;S;;;Frankfort, KY
-2;0;"Morley, Mr. Henry Samuel (""Mr Henry Marshall"")";male;39;0;0;250655;26;;S;;;
-2;0;Mudd, Mr. Thomas Charles;male;16;0;0;S.O./P.P. 3;10,5;;S;;;Halesworth, England
-2;0;Myles, Mr. Thomas Francis;male;62;0;0;240276;9,6875;;Q;;;Cambridge, MA
-2;0;Nasser, Mr. Nicholas;male;32,5;1;0;237736;30,0708;;C;;43;New York, NY
-2;1;Nasser, Mrs. Nicholas (Adele Achem);female;14;1;0;237736;30,0708;;C;;;New York, NY
-2;1;Navratil, Master. Edmond Roger;male;2;1;1;230080;26;F2;S;D;;Nice, France
-2;1;Navratil, Master. Michel M;male;3;1;1;230080;26;F2;S;D;;Nice, France
-2;0;"Navratil, Mr. Michel (""Louis M Hoffman"")";male;36,5;0;2;230080;26;F2;S;;15;Nice, France
-2;0;Nesson, Mr. Israel;male;26;0;0;244368;13;F2;S;;;Boston, MA
-2;0;Nicholls, Mr. Joseph Charles;male;19;1;1;C.A. 33112;36,75;;S;;101;Cornwall / Hancock, MI
-2;0;Norman, Mr. Robert Douglas;male;28;0;0;218629;13,5;;S;;287;Glasgow
-2;1;"Nourney, Mr. Alfred (""Baron von Drachstedt"")";male;20;0;0;SC/PARIS 2166;13,8625;D38;C;7;;Cologne, Germany
-2;1;Nye, Mrs. (Elizabeth Ramell);female;29;0;0;C.A. 29395;10,5;F33;S;11;;Folkstone, Kent / New York, NY
-2;0;Otter, Mr. Richard;male;39;0;0;28213;13;;S;;;Middleburg Heights, OH
-2;1;Oxenham, Mr. Percy Thomas;male;22;0;0;W./C. 14260;10,5;;S;13;;Pondersend, England / New Durham, NJ
-2;1;Padro y Manent, Mr. Julian;male;;0;0;SC/PARIS 2146;13,8625;;C;9;;Spain / Havana, Cuba
-2;0;Pain, Dr. Alfred;male;23;0;0;244278;10,5;;S;;;Hamilton, ON
-2;1;Pallas y Castello, Mr. Emilio;male;29;0;0;SC/PARIS 2147;13,8583;;C;9;;Spain / Havana, Cuba
-2;0;Parker, Mr. Clifford Richard;male;28;0;0;SC 14888;10,5;;S;;;St Andrews, Guernsey
-2;0;"Parkes, Mr. Francis ""Frank""";male;;0;0;239853;0;;S;;;Belfast
-2;1;Parrish, Mrs. (Lutie Davis);female;50;0;1;230433;26;;S;12;;Woodford County, KY
-2;0;Pengelly, Mr. Frederick William;male;19;0;0;28665;10,5;;S;;;Gunnislake, England / Butte, MT
-2;0;Pernot, Mr. Rene;male;;0;0;SC/PARIS 2131;15,05;;C;;;
-2;0;Peruschitz, Rev. Joseph Maria;male;41;0;0;237393;13;;S;;;
-2;1;Phillips, Miss. Alice Frances Louisa;female;21;0;1;S.O./P.P. 2;21;;S;12;;Ilfracombe, Devon
-2;1;"Phillips, Miss. Kate Florence (""Mrs Kate Louise Phillips Marshall"")";female;19;0;0;250655;26;;S;11;;Worcester, England
-2;0;Phillips, Mr. Escott Robert;male;43;0;1;S.O./P.P. 2;21;;S;;;Ilfracombe, Devon
-2;1;Pinsky, Mrs. (Rosa);female;32;0;0;234604;13;;S;9;;Russia
-2;0;Ponesell, Mr. Martin;male;34;0;0;250647;13;;S;;;Denmark / New York, NY
-2;1;Portaluppi, Mr. Emilio Ilario Giuseppe;male;30;0;0;C.A. 34644;12,7375;;C;14;;Milford, NH
-2;0;Pulbaum, Mr. Franz;male;27;0;0;SC/PARIS 2168;15,0333;;C;;;Paris
-2;1;Quick, Miss. Phyllis May;female;2;1;1;26360;26;;S;11;;Plymouth, Devon / Detroit, MI
-2;1;Quick, Miss. Winifred Vera;female;8;1;1;26360;26;;S;11;;Plymouth, Devon / Detroit, MI
-2;1;Quick, Mrs. Frederick Charles (Jane Richards);female;33;0;2;26360;26;;S;11;;Plymouth, Devon / Detroit, MI
-2;0;Reeves, Mr. David;male;36;0;0;C.A. 17248;10,5;;S;;;Brighton, Sussex
-2;0;Renouf, Mr. Peter Henry;male;34;1;0;31027;21;;S;12;;Elizabeth, NJ
-2;1;Renouf, Mrs. Peter Henry (Lillian Jefferys);female;30;3;0;31027;21;;S;;;Elizabeth, NJ
-2;1;Reynaldo, Ms. Encarnacion;female;28;0;0;230434;13;;S;9;;Spain
-2;0;Richard, Mr. Emile;male;23;0;0;SC/PARIS 2133;15,0458;;C;;;Paris / Montreal, PQ
-2;1;Richards, Master. George Sibley;male;0,8333;1;1;29106;18,75;;S;4;;Cornwall / Akron, OH
-2;1;Richards, Master. William Rowe;male;3;1;1;29106;18,75;;S;4;;Cornwall / Akron, OH
-2;1;Richards, Mrs. Sidney (Emily Hocking);female;24;2;3;29106;18,75;;S;4;;Cornwall / Akron, OH
-2;1;Ridsdale, Miss. Lucy;female;50;0;0;W./C. 14258;10,5;;S;13;;London, England / Marietta, Ohio and Milwaukee, WI
-2;0;Rogers, Mr. Reginald Harry;male;19;0;0;28004;10,5;;S;;;
-2;1;Rugg, Miss. Emily;female;21;0;0;C.A. 31026;10,5;;S;12;;Guernsey / Wilmington, DE
-2;0;Schmidt, Mr. August;male;26;0;0;248659;13;;S;;;Newark, NJ
-2;0;Sedgwick, Mr. Charles Frederick Waddington;male;25;0;0;244361;13;;S;;;Liverpool
-2;0;Sharp, Mr. Percival James R;male;27;0;0;244358;26;;S;;;Hornsey, England
-2;1;Shelley, Mrs. William (Imanita Parrish Hall);female;25;0;1;230433;26;;S;12;;Deer Lodge, MT
-2;1;Silven, Miss. Lyyli Karoliina;female;18;0;2;250652;13;;S;16;;Finland / Minneapolis, MN
-2;1;Sincock, Miss. Maude;female;20;0;0;C.A. 33112;36,75;;S;11;;Cornwall / Hancock, MI
-2;1;Sinkkonen, Miss. Anna;female;30;0;0;250648;13;;S;10;;Finland / Washington, DC
-2;0;Sjostedt, Mr. Ernst Adolf;male;59;0;0;237442;13,5;;S;;;Sault St Marie, ON
-2;1;Slayter, Miss. Hilda Mary;female;30;0;0;234818;12,35;;Q;13;;Halifax, NS
-2;0;Slemen, Mr. Richard James;male;35;0;0;28206;10,5;;S;;;Cornwall
-2;1;Smith, Miss. Marion Elsie;female;40;0;0;31418;13;;S;9;;
-2;0;Sobey, Mr. Samuel James Hayden;male;25;0;0;C.A. 29178;13;;S;;;Cornwall / Houghton, MI
-2;0;Stanton, Mr. Samuel Ward;male;41;0;0;237734;15,0458;;C;;;New York, NY
-2;0;Stokes, Mr. Philip Joseph;male;25;0;0;F.C.C. 13540;10,5;;S;;81;Catford, Kent / Detroit, MI
-2;0;Swane, Mr. George;male;18,5;0;0;248734;13;F;S;;294;
-2;0;Sweet, Mr. George Frederick;male;14;0;0;220845;65;;S;;;Somerset / Bernardsville, NJ
-2;1;Toomey, Miss. Ellen;female;50;0;0;F.C.C. 13531;10,5;;S;9;;Indianapolis, IN
-2;0;Troupiansky, Mr. Moses Aaron;male;23;0;0;233639;13;;S;;;
-2;1;Trout, Mrs. William H (Jessie L);female;28;0;0;240929;12,65;;S;;;Columbus, OH
-2;1;"Troutt, Miss. Edwina Celia ""Winnie""";female;27;0;0;34218;10,5;E101;S;16;;Bath, England / Massachusetts
-2;0;Turpin, Mr. William John Robert;male;29;1;0;11668;21;;S;;;Plymouth, England
-2;0;Turpin, Mrs. William John Robert (Dorothy Ann Wonnacott);female;27;1;0;11668;21;;S;;;Plymouth, England
-2;0;Veal, Mr. James;male;40;0;0;28221;13;;S;;;Barre, Co Washington, VT
-2;1;Walcroft, Miss. Nellie;female;31;0;0;F.C.C. 13528;21;;S;14;;Mamaroneck, NY
-2;0;Ware, Mr. John James;male;30;1;0;CA 31352;21;;S;;;Bristol, England / New Britain, CT
-2;0;Ware, Mr. William Jeffery;male;23;1;0;28666;10,5;;S;;;
-2;1;Ware, Mrs. John James (Florence Louise Long);female;31;0;0;CA 31352;21;;S;10;;Bristol, England / New Britain, CT
-2;0;Watson, Mr. Ennis Hastings;male;;0;0;239856;0;;S;;;Belfast
-2;1;Watt, Miss. Bertha J;female;12;0;0;C.A. 33595;15,75;;S;9;;Aberdeen / Portland, OR
-2;1;"Watt, Mrs. James (Elizabeth ""Bessie"" Inglis Milne)";female;40;0;0;C.A. 33595;15,75;;S;9;;Aberdeen / Portland, OR
-2;1;Webber, Miss. Susan;female;32,5;0;0;27267;13;E101;S;12;;England / Hartford, CT
-2;0;Weisz, Mr. Leopold;male;27;1;0;228414;26;;S;;293;Bromsgrove, England / Montreal, PQ
-2;1;Weisz, Mrs. Leopold (Mathilde Francoise Pede);female;29;1;0;228414;26;;S;10;;Bromsgrove, England / Montreal, PQ
-2;1;Wells, Master. Ralph Lester;male;2;1;1;29103;23;;S;14;;Cornwall / Akron, OH
-2;1;Wells, Miss. Joan;female;4;1;1;29103;23;;S;14;;Cornwall / Akron, OH
-2;1;"Wells, Mrs. Arthur Henry (""Addie"" Dart Trevaskis)";female;29;0;2;29103;23;;S;14;;Cornwall / Akron, OH
-2;1;West, Miss. Barbara J;female;0,9167;1;2;C.A. 34651;27,75;;S;10;;Bournmouth, England
-2;1;West, Miss. Constance Mirium;female;5;1;2;C.A. 34651;27,75;;S;10;;Bournmouth, England
-2;0;West, Mr. Edwy Arthur;male;36;1;2;C.A. 34651;27,75;;S;;;Bournmouth, England
-2;1;West, Mrs. Edwy Arthur (Ada Mary Worth);female;33;1;2;C.A. 34651;27,75;;S;10;;Bournmouth, England
-2;0;Wheadon, Mr. Edward H;male;66;0;0;C.A. 24579;10,5;;S;;;Guernsey, England / Edgewood, RI
-2;0;"Wheeler, Mr. Edwin ""Frederick""";male;;0;0;SC/PARIS 2159;12,875;;S;;;
-2;1;Wilhelms, Mr. Charles;male;31;0;0;244270;13;;S;9;;London, England
-2;1;Williams, Mr. Charles Eugene;male;;0;0;244373;13;;S;14;;Harrow, England
-2;1;Wright, Miss. Marion;female;26;0;0;220844;13,5;;S;9;;Yoevil, England / Cottage Grove, OR
-2;0;"Yrois, Miss. Henriette (""Mrs Harbeck"")";female;24;0;0;248747;13;;S;;;Paris
-3;0;Abbing, Mr. Anthony;male;42;0;0;C.A. 5547;7,55;;S;;;
-3;0;Abbott, Master. Eugene Joseph;male;13;0;2;C.A. 2673;20,25;;S;;;East Providence, RI
-3;0;Abbott, Mr. Rossmore Edward;male;16;1;1;C.A. 2673;20,25;;S;;190;East Providence, RI
-3;1;Abbott, Mrs. Stanton (Rosa Hunt);female;35;1;1;C.A. 2673;20,25;;S;A;;East Providence, RI
-3;1;Abelseth, Miss. Karen Marie;female;16;0;0;348125;7,65;;S;16;;Norway Los Angeles, CA
-3;1;Abelseth, Mr. Olaus Jorgensen;male;25;0;0;348122;7,65;F G63;S;A;;Perkins County, SD
-3;1;Abrahamsson, Mr. Abraham August Johannes;male;20;0;0;SOTON/O2 3101284;7,925;;S;15;;Taalintehdas, Finland Hoboken, NJ
-3;1;Abrahim, Mrs. Joseph (Sophie Halaut Easu);female;18;0;0;2657;7,2292;;C;C;;Greensburg, PA
-3;0;Adahl, Mr. Mauritz Nils Martin;male;30;0;0;C 7076;7,25;;S;;72;Asarum, Sweden Brooklyn, NY
-3;0;Adams, Mr. John;male;26;0;0;341826;8,05;;S;;103;Bournemouth, England
-3;0;Ahlin, Mrs. Johan (Johanna Persdotter Larsson);female;40;1;0;7546;9,475;;S;;;Sweden Akeley, MN
-3;1;Aks, Master. Philip Frank;male;0,8333;0;1;392091;9,35;;S;11;;London, England Norfolk, VA
-3;1;Aks, Mrs. Sam (Leah Rosen);female;18;0;1;392091;9,35;;S;13;;London, England Norfolk, VA
-3;1;Albimona, Mr. Nassef Cassem;male;26;0;0;2699;18,7875;;C;15;;Syria Fredericksburg, VA
-3;0;Alexander, Mr. William;male;26;0;0;3474;7,8875;;S;;;England Albion, NY
-3;0;Alhomaki, Mr. Ilmari Rudolf;male;20;0;0;SOTON/O2 3101287;7,925;;S;;;Salo, Finland Astoria, OR
-3;0;Ali, Mr. Ahmed;male;24;0;0;SOTON/O.Q. 3101311;7,05;;S;;;
-3;0;Ali, Mr. William;male;25;0;0;SOTON/O.Q. 3101312;7,05;;S;;79;Argentina
-3;0;Allen, Mr. William Henry;male;35;0;0;373450;8,05;;S;;;Lower Clapton, Middlesex or Erdington, Birmingham
-3;0;Allum, Mr. Owen George;male;18;0;0;2223;8,3;;S;;259;Windsor, England New York, NY
-3;0;Andersen, Mr. Albert Karvin;male;32;0;0;C 4001;22,525;;S;;260;Bergen, Norway
-3;1;Andersen-Jensen, Miss. Carla Christine Nielsine;female;19;1;0;350046;7,8542;;S;16;;
-3;0;Andersson, Master. Sigvard Harald Elias;male;4;4;2;347082;31,275;;S;;;Sweden Winnipeg, MN
-3;0;Andersson, Miss. Ebba Iris Alfrida;female;6;4;2;347082;31,275;;S;;;Sweden Winnipeg, MN
-3;0;Andersson, Miss. Ellis Anna Maria;female;2;4;2;347082;31,275;;S;;;Sweden Winnipeg, MN
-3;1;Andersson, Miss. Erna Alexandra;female;17;4;2;3101281;7,925;;S;D;;Ruotsinphyhtaa, Finland New York, NY
-3;0;Andersson, Miss. Ida Augusta Margareta;female;38;4;2;347091;7,775;;S;;;Vadsbro, Sweden Ministee, MI
-3;0;Andersson, Miss. Ingeborg Constanzia;female;9;4;2;347082;31,275;;S;;;Sweden Winnipeg, MN
-3;0;Andersson, Miss. Sigrid Elisabeth;female;11;4;2;347082;31,275;;S;;;Sweden Winnipeg, MN
-3;0;Andersson, Mr. Anders Johan;male;39;1;5;347082;31,275;;S;;;Sweden Winnipeg, MN
-3;1;"Andersson, Mr. August Edvard (""Wennerstrom"")";male;27;0;0;350043;7,7958;;S;A;;
-3;0;Andersson, Mr. Johan Samuel;male;26;0;0;347075;7,775;;S;;;Hartford, CT
-3;0;Andersson, Mrs. Anders Johan (Alfrida Konstantia Brogren);female;39;1;5;347082;31,275;;S;;;Sweden Winnipeg, MN
-3;0;Andreasson, Mr. Paul Edvin;male;20;0;0;347466;7,8542;;S;;;Sweden Chicago, IL
-3;0;Angheloff, Mr. Minko;male;26;0;0;349202;7,8958;;S;;;Bulgaria Chicago, IL
-3;0;Arnold-Franchi, Mr. Josef;male;25;1;0;349237;17,8;;S;;;Altdorf, Switzerland
-3;0;Arnold-Franchi, Mrs. Josef (Josefine Franchi);female;18;1;0;349237;17,8;;S;;;Altdorf, Switzerland
-3;0;Aronsson, Mr. Ernst Axel Algot;male;24;0;0;349911;7,775;;S;;;Sweden Joliet, IL
-3;0;Asim, Mr. Adola;male;35;0;0;SOTON/O.Q. 3101310;7,05;;S;;;
-3;0;Asplund, Master. Carl Edgar;male;5;4;2;347077;31,3875;;S;;;Sweden Worcester, MA
-3;0;Asplund, Master. Clarence Gustaf Hugo;male;9;4;2;347077;31,3875;;S;;;Sweden Worcester, MA
-3;1;Asplund, Master. Edvin Rojj Felix;male;3;4;2;347077;31,3875;;S;15;;Sweden Worcester, MA
-3;0;Asplund, Master. Filip Oscar;male;13;4;2;347077;31,3875;;S;;;Sweden Worcester, MA
-3;1;Asplund, Miss. Lillian Gertrud;female;5;4;2;347077;31,3875;;S;15;;Sweden Worcester, MA
-3;0;Asplund, Mr. Carl Oscar Vilhelm Gustafsson;male;40;1;5;347077;31,3875;;S;;142;Sweden Worcester, MA
-3;1;Asplund, Mr. Johan Charles;male;23;0;0;350054;7,7958;;S;13;;Oskarshamn, Sweden Minneapolis, MN
-3;1;Asplund, Mrs. Carl Oscar (Selma Augusta Emilia Johansson);female;38;1;5;347077;31,3875;;S;15;;Sweden Worcester, MA
-3;1;"Assaf Khalil, Mrs. Mariana (""Miriam"")";female;45;0;0;2696;7,225;;C;C;;Ottawa, ON
-3;0;Assaf, Mr. Gerios;male;21;0;0;2692;7,225;;C;;;Ottawa, ON
-3;0;Assam, Mr. Ali;male;23;0;0;SOTON/O.Q. 3101309;7,05;;S;;;
-3;0;Attalah, Miss. Malake;female;17;0;0;2627;14,4583;;C;;;
-3;0;Attalah, Mr. Sleiman;male;30;0;0;2694;7,225;;C;;;Ottawa, ON
-3;0;Augustsson, Mr. Albert;male;23;0;0;347468;7,8542;;S;;;Krakoryd, Sweden Bloomington, IL
-3;1;Ayoub, Miss. Banoura;female;13;0;0;2687;7,2292;;C;C;;Syria Youngstown, OH
-3;0;Baccos, Mr. Raffull;male;20;0;0;2679;7,225;;C;;;
-3;0;Backstrom, Mr. Karl Alfred;male;32;1;0;3101278;15,85;;S;D;;Ruotsinphytaa, Finland New York, NY
-3;1;Backstrom, Mrs. Karl Alfred (Maria Mathilda Gustafsson);female;33;3;0;3101278;15,85;;S;;;Ruotsinphytaa, Finland New York, NY
-3;1;Baclini, Miss. Eugenie;female;0,75;2;1;2666;19,2583;;C;C;;Syria New York, NY
-3;1;Baclini, Miss. Helene Barbara;female;0,75;2;1;2666;19,2583;;C;C;;Syria New York, NY
-3;1;Baclini, Miss. Marie Catherine;female;5;2;1;2666;19,2583;;C;C;;Syria New York, NY
-3;1;Baclini, Mrs. Solomon (Latifa Qurban);female;24;0;3;2666;19,2583;;C;C;;Syria New York, NY
-3;1;Badman, Miss. Emily Louisa;female;18;0;0;A/4 31416;8,05;;S;C;;London Skanteales, NY
-3;0;Badt, Mr. Mohamed;male;40;0;0;2623;7,225;;C;;;
-3;0;Balkic, Mr. Cerin;male;26;0;0;349248;7,8958;;S;;;
-3;1;Barah, Mr. Hanna Assi;male;20;0;0;2663;7,2292;;C;15;;
-3;0;Barbara, Miss. Saiide;female;18;0;1;2691;14,4542;;C;;;Syria Ottawa, ON
-3;0;Barbara, Mrs. (Catherine David);female;45;0;1;2691;14,4542;;C;;;Syria Ottawa, ON
-3;0;Barry, Miss. Julia;female;27;0;0;330844;7,8792;;Q;;;New York, NY
-3;0;Barton, Mr. David John;male;22;0;0;324669;8,05;;S;;;England New York, NY
-3;0;Beavan, Mr. William Thomas;male;19;0;0;323951;8,05;;S;;;England
-3;0;Bengtsson, Mr. John Viktor;male;26;0;0;347068;7,775;;S;;;Krakudden, Sweden Moune, IL
-3;0;Berglund, Mr. Karl Ivar Sven;male;22;0;0;PP 4348;9,35;;S;;;Tranvik, Finland New York
-3;0;Betros, Master. Seman;male;;0;0;2622;7,2292;;C;;;
-3;0;Betros, Mr. Tannous;male;20;0;0;2648;4,0125;;C;;;Syria
-3;1;Bing, Mr. Lee;male;32;0;0;1601;56,4958;;S;C;;Hong Kong New York, NY
-3;0;Birkeland, Mr. Hans Martin Monsen;male;21;0;0;312992;7,775;;S;;;Brennes, Norway New York
-3;0;Bjorklund, Mr. Ernst Herbert;male;18;0;0;347090;7,75;;S;;;Stockholm, Sweden New York
-3;0;Bostandyeff, Mr. Guentcho;male;26;0;0;349224;7,8958;;S;;;Bulgaria Chicago, IL
-3;0;Boulos, Master. Akar;male;6;1;1;2678;15,2458;;C;;;Syria Kent, ON
-3;0;Boulos, Miss. Nourelain;female;9;1;1;2678;15,2458;;C;;;Syria Kent, ON
-3;0;Boulos, Mr. Hanna;male;;0;0;2664;7,225;;C;;;Syria
-3;0;Boulos, Mrs. Joseph (Sultana);female;;0;2;2678;15,2458;;C;;;Syria Kent, ON
-3;0;Bourke, Miss. Mary;female;;0;2;364848;7,75;;Q;;;Ireland Chicago, IL
-3;0;Bourke, Mr. John;male;40;1;1;364849;15,5;;Q;;;Ireland Chicago, IL
-3;0;Bourke, Mrs. John (Catherine);female;32;1;1;364849;15,5;;Q;;;Ireland Chicago, IL
-3;0;"Bowen, Mr. David John ""Dai""";male;21;0;0;54636;16,1;;S;;;Treherbert, Cardiff, Wales
-3;1;Bradley, Miss. Bridget Delia;female;22;0;0;334914;7,725;;Q;13;;Kingwilliamstown, Co Cork, Ireland Glens Falls, NY
-3;0;Braf, Miss. Elin Ester Maria;female;20;0;0;347471;7,8542;;S;;;Medeltorp, Sweden Chicago, IL
-3;0;Braund, Mr. Lewis Richard;male;29;1;0;3460;7,0458;;S;;;Bridgerule, Devon
-3;0;Braund, Mr. Owen Harris;male;22;1;0;A/5 21171;7,25;;S;;;Bridgerule, Devon
-3;0;Brobeck, Mr. Karl Rudolf;male;22;0;0;350045;7,7958;;S;;;Sweden Worcester, MA
-3;0;Brocklebank, Mr. William Alfred;male;35;0;0;364512;8,05;;S;;;Broomfield, Chelmsford, England
-3;0;Buckley, Miss. Katherine;female;18,5;0;0;329944;7,2833;;Q;;299;Co Cork, Ireland Roxbury, MA
-3;1;Buckley, Mr. Daniel;male;21;0;0;330920;7,8208;;Q;13;;Kingwilliamstown, Co Cork, Ireland New York, NY
-3;0;Burke, Mr. Jeremiah;male;19;0;0;365222;6,75;;Q;;;Co Cork, Ireland Charlestown, MA
-3;0;Burns, Miss. Mary Delia;female;18;0;0;330963;7,8792;;Q;;;Co Sligo, Ireland New York, NY
-3;0;Cacic, Miss. Manda;female;21;0;0;315087;8,6625;;S;;;
-3;0;Cacic, Miss. Marija;female;30;0;0;315084;8,6625;;S;;;
-3;0;Cacic, Mr. Jego Grga;male;18;0;0;315091;8,6625;;S;;;
-3;0;Cacic, Mr. Luka;male;38;0;0;315089;8,6625;;S;;;Croatia
-3;0;Calic, Mr. Jovo;male;17;0;0;315093;8,6625;;S;;;
-3;0;Calic, Mr. Petar;male;17;0;0;315086;8,6625;;S;;;
-3;0;Canavan, Miss. Mary;female;21;0;0;364846;7,75;;Q;;;
-3;0;Canavan, Mr. Patrick;male;21;0;0;364858;7,75;;Q;;;Ireland Philadelphia, PA
-3;0;Cann, Mr. Ernest Charles;male;21;0;0;A./5. 2152;8,05;;S;;;
-3;0;Caram, Mr. Joseph;male;;1;0;2689;14,4583;;C;;;Ottawa, ON
-3;0;Caram, Mrs. Joseph (Maria Elias);female;;1;0;2689;14,4583;;C;;;Ottawa, ON
-3;0;Carlsson, Mr. August Sigfrid;male;28;0;0;350042;7,7958;;S;;;Dagsas, Sweden Fower, MN
-3;0;Carlsson, Mr. Carl Robert;male;24;0;0;350409;7,8542;;S;;;Goteborg, Sweden Huntley, IL
-3;1;"Carr, Miss. Helen ""Ellen""";female;16;0;0;367231;7,75;;Q;16;;Co Longford, Ireland New York, NY
-3;0;Carr, Miss. Jeannie;female;37;0;0;368364;7,75;;Q;;;Co Sligo, Ireland Hartford, CT
-3;0;Carver, Mr. Alfred John;male;28;0;0;392095;7,25;;S;;;St Denys, Southampton, Hants
-3;0;Celotti, Mr. Francesco;male;24;0;0;343275;8,05;;S;;;London
-3;0;Charters, Mr. David;male;21;0;0;A/5. 13032;7,7333;;Q;;;Ireland New York, NY
-3;1;Chip, Mr. Chang;male;32;0;0;1601;56,4958;;S;C;;Hong Kong New York, NY
-3;0;Christmann, Mr. Emil;male;29;0;0;343276;8,05;;S;;;
-3;0;Chronopoulos, Mr. Apostolos;male;26;1;0;2680;14,4542;;C;;;Greece
-3;0;Chronopoulos, Mr. Demetrios;male;18;1;0;2680;14,4542;;C;;;Greece
-3;0;Coelho, Mr. Domingos Fernandeo;male;20;0;0;SOTON/O.Q. 3101307;7,05;;S;;;Portugal
-3;1;"Cohen, Mr. Gurshon ""Gus""";male;18;0;0;A/5 3540;8,05;;S;12;;London Brooklyn, NY
-3;0;Colbert, Mr. Patrick;male;24;0;0;371109;7,25;;Q;;;Co Limerick, Ireland Sherbrooke, PQ
-3;0;Coleff, Mr. Peju;male;36;0;0;349210;7,4958;;S;;;Bulgaria Chicago, IL
-3;0;Coleff, Mr. Satio;male;24;0;0;349209;7,4958;;S;;;
-3;0;Conlon, Mr. Thomas Henry;male;31;0;0;21332;7,7333;;Q;;;Philadelphia, PA
-3;0;Connaghton, Mr. Michael;male;31;0;0;335097;7,75;;Q;;;Ireland Brooklyn, NY
-3;1;Connolly, Miss. Kate;female;22;0;0;370373;7,75;;Q;13;;Ireland
-3;0;Connolly, Miss. Kate;female;30;0;0;330972;7,6292;;Q;;;Ireland
-3;0;Connors, Mr. Patrick;male;70,5;0;0;370369;7,75;;Q;;171;
-3;0;Cook, Mr. Jacob;male;43;0;0;A/5 3536;8,05;;S;;;
-3;0;Cor, Mr. Bartol;male;35;0;0;349230;7,8958;;S;;;Austria
-3;0;Cor, Mr. Ivan;male;27;0;0;349229;7,8958;;S;;;Austria
-3;0;Cor, Mr. Liudevit;male;19;0;0;349231;7,8958;;S;;;Austria
-3;0;Corn, Mr. Harry;male;30;0;0;SOTON/OQ 392090;8,05;;S;;;London
-3;1;"Coutts, Master. Eden Leslie ""Neville""";male;9;1;1;C.A. 37671;15,9;;S;2;;England Brooklyn, NY
-3;1;"Coutts, Master. William Loch ""William""";male;3;1;1;C.A. 37671;15,9;;S;2;;England Brooklyn, NY
-3;1;"Coutts, Mrs. William (Winnie ""Minnie"" Treanor)";female;36;0;2;C.A. 37671;15,9;;S;2;;England Brooklyn, NY
-3;0;Coxon, Mr. Daniel;male;59;0;0;364500;7,25;;S;;;Merrill, WI
-3;0;Crease, Mr. Ernest James;male;19;0;0;S.P. 3464;8,1583;;S;;;Bristol, England Cleveland, OH
-3;1;Cribb, Miss. Laura Alice;female;17;0;1;371362;16,1;;S;12;;Bournemouth, England Newark, NJ
-3;0;Cribb, Mr. John Hatfield;male;44;0;1;371362;16,1;;S;;;Bournemouth, England Newark, NJ
-3;0;Culumovic, Mr. Jeso;male;17;0;0;315090;8,6625;;S;;;Austria-Hungary
-3;0;Daher, Mr. Shedid;male;22,5;0;0;2698;7,225;;C;;9;
-3;1;Dahl, Mr. Karl Edwart;male;45;0;0;7598;8,05;;S;15;;Australia Fingal, ND
-3;0;Dahlberg, Miss. Gerda Ulrika;female;22;0;0;7552;10,5167;;S;;;Norrlot, Sweden Chicago, IL
-3;0;Dakic, Mr. Branko;male;19;0;0;349228;10,1708;;S;;;Austria
-3;1;"Daly, Miss. Margaret Marcella ""Maggie""";female;30;0;0;382650;6,95;;Q;15;;Co Athlone, Ireland New York, NY
-3;1;Daly, Mr. Eugene Patrick;male;29;0;0;382651;7,75;;Q;13 15 B;;Co Athlone, Ireland New York, NY
-3;0;Danbom, Master. Gilbert Sigvard Emanuel;male;0,3333;0;2;347080;14,4;;S;;;Stanton, IA
-3;0;Danbom, Mr. Ernst Gilbert;male;34;1;1;347080;14,4;;S;;197;Stanton, IA
-3;0;Danbom, Mrs. Ernst Gilbert (Anna Sigrid Maria Brogren);female;28;1;1;347080;14,4;;S;;;Stanton, IA
-3;0;Danoff, Mr. Yoto;male;27;0;0;349219;7,8958;;S;;;Bulgaria Chicago, IL
-3;0;Dantcheff, Mr. Ristiu;male;25;0;0;349203;7,8958;;S;;;Bulgaria Chicago, IL
-3;0;Davies, Mr. Alfred J;male;24;2;0;A/4 48871;24,15;;S;;;West Bromwich, England Pontiac, MI
-3;0;Davies, Mr. Evan;male;22;0;0;SC/A4 23568;8,05;;S;;;
-3;0;Davies, Mr. John Samuel;male;21;2;0;A/4 48871;24,15;;S;;;West Bromwich, England Pontiac, MI
-3;0;Davies, Mr. Joseph;male;17;2;0;A/4 48873;8,05;;S;;;West Bromwich, England Pontiac, MI
-3;0;Davison, Mr. Thomas Henry;male;;1;0;386525;16,1;;S;;;Liverpool, England Bedford, OH
-3;1;Davison, Mrs. Thomas Henry (Mary E Finck);female;;1;0;386525;16,1;;S;16;;Liverpool, England Bedford, OH
-3;1;de Messemaeker, Mr. Guillaume Joseph;male;36,5;1;0;345572;17,4;;S;15;;Tampico, MT
-3;1;de Messemaeker, Mrs. Guillaume Joseph (Emma);female;36;1;0;345572;17,4;;S;13;;Tampico, MT
-3;1;de Mulder, Mr. Theodore;male;30;0;0;345774;9,5;;S;11;;Belgium Detroit, MI
-3;0;de Pelsmaeker, Mr. Alfons;male;16;0;0;345778;9,5;;S;;;
-3;1;Dean, Master. Bertram Vere;male;1;1;2;C.A. 2315;20,575;;S;10;;Devon, England Wichita, KS
-3;1;"Dean, Miss. Elizabeth Gladys ""Millvina""";female;0,1667;1;2;C.A. 2315;20,575;;S;10;;Devon, England Wichita, KS
-3;0;Dean, Mr. Bertram Frank;male;26;1;2;C.A. 2315;20,575;;S;;;Devon, England Wichita, KS
-3;1;Dean, Mrs. Bertram (Eva Georgetta Light);female;33;1;2;C.A. 2315;20,575;;S;10;;Devon, England Wichita, KS
-3;0;Delalic, Mr. Redjo;male;25;0;0;349250;7,8958;;S;;;
-3;0;Demetri, Mr. Marinko;male;;0;0;349238;7,8958;;S;;;
-3;0;Denkoff, Mr. Mitto;male;;0;0;349225;7,8958;;S;;;Bulgaria Coon Rapids, IA
-3;0;Dennis, Mr. Samuel;male;22;0;0;A/5 21172;7,25;;S;;;
-3;0;Dennis, Mr. William;male;36;0;0;A/5 21175;7,25;;S;;;
-3;1;Devaney, Miss. Margaret Delia;female;19;0;0;330958;7,8792;;Q;C;;Kilmacowen, Co Sligo, Ireland New York, NY
-3;0;Dika, Mr. Mirko;male;17;0;0;349232;7,8958;;S;;;
-3;0;Dimic, Mr. Jovan;male;42;0;0;315088;8,6625;;S;;;
-3;0;Dintcheff, Mr. Valtcho;male;43;0;0;349226;7,8958;;S;;;
-3;0;Doharr, Mr. Tannous;male;;0;0;2686;7,2292;;C;;;
-3;0;Dooley, Mr. Patrick;male;32;0;0;370376;7,75;;Q;;;Ireland New York, NY
-3;1;Dorking, Mr. Edward Arthur;male;19;0;0;A/5. 10482;8,05;;S;B;;England Oglesby, IL
-3;1;Dowdell, Miss. Elizabeth;female;30;0;0;364516;12,475;;S;13;;Union Hill, NJ
-3;0;Doyle, Miss. Elizabeth;female;24;0;0;368702;7,75;;Q;;;Ireland New York, NY
-3;1;Drapkin, Miss. Jennie;female;23;0;0;SOTON/OQ 392083;8,05;;S;;;London New York, NY
-3;0;Drazenoic, Mr. Jozef;male;33;0;0;349241;7,8958;;C;;51;Austria Niagara Falls, NY
-3;0;Duane, Mr. Frank;male;65;0;0;336439;7,75;;Q;;;
-3;1;Duquemin, Mr. Joseph;male;24;0;0;S.O./P.P. 752;7,55;;S;D;;England Albion, NY
-3;0;Dyker, Mr. Adolf Fredrik;male;23;1;0;347072;13,9;;S;;;West Haven, CT
-3;1;Dyker, Mrs. Adolf Fredrik (Anna Elisabeth Judith Andersson);female;22;1;0;347072;13,9;;S;16;;West Haven, CT
-3;0;Edvardsson, Mr. Gustaf Hjalmar;male;18;0;0;349912;7,775;;S;;;Tofta, Sweden Joliet, IL
-3;0;Eklund, Mr. Hans Linus;male;16;0;0;347074;7,775;;S;;;Karberg, Sweden Jerome Junction, AZ
-3;0;Ekstrom, Mr. Johan;male;45;0;0;347061;6,975;;S;;;Effington Rut, SD
-3;0;Elias, Mr. Dibo;male;;0;0;2674;7,225;;C;;;
-3;0;Elias, Mr. Joseph;male;39;0;2;2675;7,2292;;C;;;Syria Ottawa, ON
-3;0;Elias, Mr. Joseph Jr;male;17;1;1;2690;7,2292;;C;;;
-3;0;Elias, Mr. Tannous;male;15;1;1;2695;7,2292;;C;;;Syria
-3;0;Elsbury, Mr. William James;male;47;0;0;A/5 3902;7,25;;S;;;Illinois, USA
-3;1;Emanuel, Miss. Virginia Ethel;female;5;0;0;364516;12,475;;S;13;;New York, NY
-3;0;Emir, Mr. Farred Chehab;male;;0;0;2631;7,225;;C;;;
-3;0;Everett, Mr. Thomas James;male;40,5;0;0;C.A. 6212;15,1;;S;;187;
-3;0;Farrell, Mr. James;male;40,5;0;0;367232;7,75;;Q;;68;Aughnacliff, Co Longford, Ireland New York, NY
-3;1;Finoli, Mr. Luigi;male;;0;0;SOTON/O.Q. 3101308;7,05;;S;15;;Italy Philadelphia, PA
-3;0;Fischer, Mr. Eberhard Thelander;male;18;0;0;350036;7,7958;;S;;;
-3;0;Fleming, Miss. Honora;female;;0;0;364859;7,75;;Q;;;
-3;0;Flynn, Mr. James;male;;0;0;364851;7,75;;Q;;;
-3;0;Flynn, Mr. John;male;;0;0;368323;6,95;;Q;;;
-3;0;Foley, Mr. Joseph;male;26;0;0;330910;7,8792;;Q;;;Ireland Chicago, IL
-3;0;Foley, Mr. William;male;;0;0;365235;7,75;;Q;;;Ireland
-3;1;Foo, Mr. Choong;male;;0;0;1601;56,4958;;S;13;;Hong Kong New York, NY
-3;0;"Ford, Miss. Doolina Margaret ""Daisy""";female;21;2;2;W./C. 6608;34,375;;S;;;Rotherfield, Sussex, England Essex Co, MA
-3;0;"Ford, Miss. Robina Maggie ""Ruby""";female;9;2;2;W./C. 6608;34,375;;S;;;Rotherfield, Sussex, England Essex Co, MA
-3;0;Ford, Mr. Arthur;male;;0;0;A/5 1478;8,05;;S;;;Bridgwater, Somerset, England
-3;0;Ford, Mr. Edward Watson;male;18;2;2;W./C. 6608;34,375;;S;;;Rotherfield, Sussex, England Essex Co, MA
-3;0;Ford, Mr. William Neal;male;16;1;3;W./C. 6608;34,375;;S;;;Rotherfield, Sussex, England Essex Co, MA
-3;0;Ford, Mrs. Edward (Margaret Ann Watson);female;48;1;3;W./C. 6608;34,375;;S;;;Rotherfield, Sussex, England Essex Co, MA
-3;0;Fox, Mr. Patrick;male;;0;0;368573;7,75;;Q;;;Ireland New York, NY
-3;0;Franklin, Mr. Charles (Charles Fardon);male;;0;0;SOTON/O.Q. 3101314;7,25;;S;;;
-3;0;Gallagher, Mr. Martin;male;25;0;0;36864;7,7417;;Q;;;New York, NY
-3;0;Garfirth, Mr. John;male;;0;0;358585;14,5;;S;;;
-3;0;Gheorgheff, Mr. Stanio;male;;0;0;349254;7,8958;;C;;;
-3;0;Gilinski, Mr. Eliezer;male;22;0;0;14973;8,05;;S;;47;
-3;1;"Gilnagh, Miss. Katherine ""Katie""";female;16;0;0;35851;7,7333;;Q;16;;Co Longford, Ireland New York, NY
-3;1;Glynn, Miss. Mary Agatha;female;;0;0;335677;7,75;;Q;13;;Co Clare, Ireland Washington, DC
-3;1;"Goldsmith, Master. Frank John William ""Frankie""";male;9;0;2;363291;20,525;;S;C D;;Strood, Kent, England Detroit, MI
-3;0;Goldsmith, Mr. Frank John;male;33;1;1;363291;20,525;;S;;;Strood, Kent, England Detroit, MI
-3;0;Goldsmith, Mr. Nathan;male;41;0;0;SOTON/O.Q. 3101263;7,85;;S;;;Philadelphia, PA
-3;1;Goldsmith, Mrs. Frank John (Emily Alice Brown);female;31;1;1;363291;20,525;;S;C D;;Strood, Kent, England Detroit, MI
-3;0;Goncalves, Mr. Manuel Estanslas;male;38;0;0;SOTON/O.Q. 3101306;7,05;;S;;;Portugal
-3;0;Goodwin, Master. Harold Victor;male;9;5;2;CA 2144;46,9;;S;;;Wiltshire, England Niagara Falls, NY
-3;0;Goodwin, Master. Sidney Leonard;male;1;5;2;CA 2144;46,9;;S;;;Wiltshire, England Niagara Falls, NY
-3;0;Goodwin, Master. William Frederick;male;11;5;2;CA 2144;46,9;;S;;;Wiltshire, England Niagara Falls, NY
-3;0;Goodwin, Miss. Jessie Allis;female;10;5;2;CA 2144;46,9;;S;;;Wiltshire, England Niagara Falls, NY
-3;0;Goodwin, Miss. Lillian Amy;female;16;5;2;CA 2144;46,9;;S;;;Wiltshire, England Niagara Falls, NY
-3;0;Goodwin, Mr. Charles Edward;male;14;5;2;CA 2144;46,9;;S;;;Wiltshire, England Niagara Falls, NY
-3;0;Goodwin, Mr. Charles Frederick;male;40;1;6;CA 2144;46,9;;S;;;Wiltshire, England Niagara Falls, NY
-3;0;Goodwin, Mrs. Frederick (Augusta Tyler);female;43;1;6;CA 2144;46,9;;S;;;Wiltshire, England Niagara Falls, NY
-3;0;Green, Mr. George Henry;male;51;0;0;21440;8,05;;S;;;Dorking, Surrey, England
-3;0;Gronnestad, Mr. Daniel Danielsen;male;32;0;0;8471;8,3625;;S;;;Foresvik, Norway Portland, ND
-3;0;Guest, Mr. Robert;male;;0;0;376563;8,05;;S;;;
-3;0;Gustafsson, Mr. Alfred Ossian;male;20;0;0;7534;9,8458;;S;;;Waukegan, Chicago, IL
-3;0;Gustafsson, Mr. Anders Vilhelm;male;37;2;0;3101276;7,925;;S;;98;Ruotsinphytaa, Finland New York, NY
-3;0;Gustafsson, Mr. Johan Birger;male;28;2;0;3101277;7,925;;S;;;Ruotsinphytaa, Finland New York, NY
-3;0;Gustafsson, Mr. Karl Gideon;male;19;0;0;347069;7,775;;S;;;Myren, Sweden New York, NY
-3;0;Haas, Miss. Aloisia;female;24;0;0;349236;8,85;;S;;;
-3;0;Hagardon, Miss. Kate;female;17;0;0;AQ/3. 30631;7,7333;;Q;;;
-3;0;Hagland, Mr. Ingvald Olai Olsen;male;;1;0;65303;19,9667;;S;;;
-3;0;Hagland, Mr. Konrad Mathias Reiersen;male;;1;0;65304;19,9667;;S;;;
-3;0;Hakkarainen, Mr. Pekka Pietari;male;28;1;0;STON/O2. 3101279;15,85;;S;;;
-3;1;Hakkarainen, Mrs. Pekka Pietari (Elin Matilda Dolck);female;24;1;0;STON/O2. 3101279;15,85;;S;15;;
-3;0;Hampe, Mr. Leon;male;20;0;0;345769;9,5;;S;;;
-3;0;Hanna, Mr. Mansour;male;23,5;0;0;2693;7,2292;;C;;188;
-3;0;Hansen, Mr. Claus Peter;male;41;2;0;350026;14,1083;;S;;;
-3;0;Hansen, Mr. Henrik Juul;male;26;1;0;350025;7,8542;;S;;;
-3;0;Hansen, Mr. Henry Damsgaard;male;21;0;0;350029;7,8542;;S;;69;
-3;1;Hansen, Mrs. Claus Peter (Jennie L Howard);female;45;1;0;350026;14,1083;;S;11;;
-3;0;Harknett, Miss. Alice Phoebe;female;;0;0;W./C. 6609;7,55;;S;;;
-3;0;Harmer, Mr. Abraham (David Lishin);male;25;0;0;374887;7,25;;S;B;;
-3;0;Hart, Mr. Henry;male;;0;0;394140;6,8583;;Q;;;
-3;0;Hassan, Mr. Houssein G N;male;11;0;0;2699;18,7875;;C;;;
-3;1;"Healy, Miss. Hanora ""Nora""";female;;0;0;370375;7,75;;Q;16;;
-3;1;Hedman, Mr. Oskar Arvid;male;27;0;0;347089;6,975;;S;15;;
-3;1;Hee, Mr. Ling;male;;0;0;1601;56,4958;;S;C;;
-3;0;"Hegarty, Miss. Hanora ""Nora""";female;18;0;0;365226;6,75;;Q;;;
-3;1;Heikkinen, Miss. Laina;female;26;0;0;STON/O2. 3101282;7,925;;S;;;
-3;0;Heininen, Miss. Wendla Maria;female;23;0;0;STON/O2. 3101290;7,925;;S;;;
-3;1;Hellstrom, Miss. Hilda Maria;female;22;0;0;7548;8,9625;;S;C;;
-3;0;Hendekovic, Mr. Ignjac;male;28;0;0;349243;7,8958;;S;;306;
-3;0;Henriksson, Miss. Jenny Lovisa;female;28;0;0;347086;7,775;;S;;;
-3;0;Henry, Miss. Delia;female;;0;0;382649;7,75;;Q;;;
-3;1;Hirvonen, Miss. Hildur E;female;2;0;1;3101298;12,2875;;S;15;;
-3;1;Hirvonen, Mrs. Alexander (Helga E Lindqvist);female;22;1;1;3101298;12,2875;;S;15;;
-3;0;Holm, Mr. John Fredrik Alexander;male;43;0;0;C 7075;6,45;;S;;;
-3;0;Holthen, Mr. Johan Martin;male;28;0;0;C 4001;22,525;;S;;;
-3;1;Honkanen, Miss. Eliina;female;27;0;0;STON/O2. 3101283;7,925;;S;;;
-3;0;Horgan, Mr. John;male;;0;0;370377;7,75;;Q;;;
-3;1;Howard, Miss. May Elizabeth;female;;0;0;A. 2. 39186;8,05;;S;C;;
-3;0;Humblen, Mr. Adolf Mathias Nicolai Olsen;male;42;0;0;348121;7,65;F G63;S;;120;
-3;1;Hyman, Mr. Abraham;male;;0;0;3470;7,8875;;S;C;;
-3;0;Ibrahim Shawah, Mr. Yousseff;male;30;0;0;2685;7,2292;;C;;;
-3;0;Ilieff, Mr. Ylio;male;;0;0;349220;7,8958;;S;;;
-3;0;Ilmakangas, Miss. Ida Livija;female;27;1;0;STON/O2. 3101270;7,925;;S;;;
-3;0;Ilmakangas, Miss. Pieta Sofia;female;25;1;0;STON/O2. 3101271;7,925;;S;;;
-3;0;Ivanoff, Mr. Kanio;male;;0;0;349201;7,8958;;S;;;
-3;1;Jalsevac, Mr. Ivan;male;29;0;0;349240;7,8958;;C;15;;
-3;1;Jansson, Mr. Carl Olof;male;21;0;0;350034;7,7958;;S;A;;
-3;0;Jardin, Mr. Jose Neto;male;;0;0;SOTON/O.Q. 3101305;7,05;;S;;;
-3;0;Jensen, Mr. Hans Peder;male;20;0;0;350050;7,8542;;S;;;
-3;0;Jensen, Mr. Niels Peder;male;48;0;0;350047;7,8542;;S;;;
-3;0;Jensen, Mr. Svend Lauritz;male;17;1;0;350048;7,0542;;S;;;
-3;1;Jermyn, Miss. Annie;female;;0;0;14313;7,75;;Q;D;;
-3;1;Johannesen-Bratthammer, Mr. Bernt;male;;0;0;65306;8,1125;;S;13;;
-3;0;Johanson, Mr. Jakob Alfred;male;34;0;0;3101264;6,4958;;S;;143;
-3;1;Johansson Palmquist, Mr. Oskar Leander;male;26;0;0;347070;7,775;;S;15;;
-3;0;Johansson, Mr. Erik;male;22;0;0;350052;7,7958;;S;;156;
-3;0;Johansson, Mr. Gustaf Joel;male;33;0;0;7540;8,6542;;S;;285;
-3;0;Johansson, Mr. Karl Johan;male;31;0;0;347063;7,775;;S;;;
-3;0;Johansson, Mr. Nils;male;29;0;0;347467;7,8542;;S;;;
-3;1;Johnson, Master. Harold Theodor;male;4;1;1;347742;11,1333;;S;15;;
-3;1;Johnson, Miss. Eleanor Ileen;female;1;1;1;347742;11,1333;;S;15;;
-3;0;Johnson, Mr. Alfred;male;49;0;0;LINE;0;;S;;;
-3;0;Johnson, Mr. Malkolm Joackim;male;33;0;0;347062;7,775;;S;;37;
-3;0;Johnson, Mr. William Cahoone Jr;male;19;0;0;LINE;0;;S;;;
-3;1;Johnson, Mrs. Oscar W (Elisabeth Vilhelmina Berg);female;27;0;2;347742;11,1333;;S;15;;
-3;0;"Johnston, Master. William Arthur ""Willie""";male;;1;2;W./C. 6607;23,45;;S;;;
-3;0;"Johnston, Miss. Catherine Helen ""Carrie""";female;;1;2;W./C. 6607;23,45;;S;;;
-3;0;Johnston, Mr. Andrew G;male;;1;2;W./C. 6607;23,45;;S;;;
-3;0;"Johnston, Mrs. Andrew G (Elizabeth ""Lily"" Watson)";female;;1;2;W./C. 6607;23,45;;S;;;
-3;0;Jonkoff, Mr. Lalio;male;23;0;0;349204;7,8958;;S;;;
-3;1;Jonsson, Mr. Carl;male;32;0;0;350417;7,8542;;S;15;;
-3;0;Jonsson, Mr. Nils Hilding;male;27;0;0;350408;7,8542;;S;;;
-3;0;Jussila, Miss. Katriina;female;20;1;0;4136;9,825;;S;;;
-3;0;Jussila, Miss. Mari Aina;female;21;1;0;4137;9,825;;S;;;
-3;1;Jussila, Mr. Eiriik;male;32;0;0;STON/O 2. 3101286;7,925;;S;15;;
-3;0;Kallio, Mr. Nikolai Erland;male;17;0;0;STON/O 2. 3101274;7,125;;S;;;
-3;0;Kalvik, Mr. Johannes Halvorsen;male;21;0;0;8475;8,4333;;S;;;
-3;0;Karaic, Mr. Milan;male;30;0;0;349246;7,8958;;S;;;
-3;1;Karlsson, Mr. Einar Gervasius;male;21;0;0;350053;7,7958;;S;13;;
-3;0;Karlsson, Mr. Julius Konrad Eugen;male;33;0;0;347465;7,8542;;S;;;
-3;0;Karlsson, Mr. Nils August;male;22;0;0;350060;7,5208;;S;;;
-3;1;Karun, Miss. Manca;female;4;0;1;349256;13,4167;;C;15;;
-3;1;Karun, Mr. Franz;male;39;0;1;349256;13,4167;;C;15;;
-3;0;Kassem, Mr. Fared;male;;0;0;2700;7,2292;;C;;;
-3;0;"Katavelas, Mr. Vassilios (""Catavelas Vassilios"")";male;18,5;0;0;2682;7,2292;;C;;58;
-3;0;"Keane, Mr. Andrew ""Andy""";male;;0;0;12460;7,75;;Q;;;
-3;0;Keefe, Mr. Arthur;male;;0;0;323592;7,25;;S;A;;
-3;1;"Kelly, Miss. Anna Katherine ""Annie Kate""";female;;0;0;9234;7,75;;Q;16;;
-3;1;Kelly, Miss. Mary;female;;0;0;14312;7,75;;Q;D;;
-3;0;Kelly, Mr. James;male;34,5;0;0;330911;7,8292;;Q;;70;
-3;0;Kelly, Mr. James;male;44;0;0;363592;8,05;;S;;;
-3;1;Kennedy, Mr. John;male;;0;0;368783;7,75;;Q;;;
-3;0;Khalil, Mr. Betros;male;;1;0;2660;14,4542;;C;;;
-3;0;"Khalil, Mrs. Betros (Zahie ""Maria"" Elias)";female;;1;0;2660;14,4542;;C;;;
-3;0;Kiernan, Mr. John;male;;1;0;367227;7,75;;Q;;;
-3;0;Kiernan, Mr. Philip;male;;1;0;367229;7,75;;Q;;;
-3;0;Kilgannon, Mr. Thomas J;male;;0;0;36865;7,7375;;Q;;;
-3;0;Kink, Miss. Maria;female;22;2;0;315152;8,6625;;S;;;
-3;0;Kink, Mr. Vincenz;male;26;2;0;315151;8,6625;;S;;;
-3;1;Kink-Heilmann, Miss. Luise Gretchen;female;4;0;2;315153;22,025;;S;2;;
-3;1;Kink-Heilmann, Mr. Anton;male;29;3;1;315153;22,025;;S;2;;
-3;1;Kink-Heilmann, Mrs. Anton (Luise Heilmann);female;26;1;1;315153;22,025;;S;2;;
-3;0;Klasen, Miss. Gertrud Emilia;female;1;1;1;350405;12,1833;;S;;;
-3;0;Klasen, Mr. Klas Albin;male;18;1;1;350404;7,8542;;S;;;
-3;0;Klasen, Mrs. (Hulda Kristina Eugenia Lofqvist);female;36;0;2;350405;12,1833;;S;;;
-3;0;Kraeff, Mr. Theodor;male;;0;0;349253;7,8958;;C;;;
-3;1;Krekorian, Mr. Neshan;male;25;0;0;2654;7,2292;F E57;C;10;;
-3;0;Lahoud, Mr. Sarkis;male;;0;0;2624;7,225;;C;;;
-3;0;Laitinen, Miss. Kristina Sofia;female;37;0;0;4135;9,5875;;S;;;
-3;0;Laleff, Mr. Kristo;male;;0;0;349217;7,8958;;S;;;
-3;1;Lam, Mr. Ali;male;;0;0;1601;56,4958;;S;C;;
-3;0;Lam, Mr. Len;male;;0;0;1601;56,4958;;S;;;
-3;1;Landergren, Miss. Aurora Adelia;female;22;0;0;C 7077;7,25;;S;13;;
-3;0;Lane, Mr. Patrick;male;;0;0;7935;7,75;;Q;;;
-3;1;Lang, Mr. Fang;male;26;0;0;1601;56,4958;;S;14;;
-3;0;Larsson, Mr. August Viktor;male;29;0;0;7545;9,4833;;S;;;
-3;0;Larsson, Mr. Bengt Edvin;male;29;0;0;347067;7,775;;S;;;
-3;0;Larsson-Rondberg, Mr. Edvard A;male;22;0;0;347065;7,775;;S;;;
-3;1;"Leeni, Mr. Fahim (""Philip Zenni"")";male;22;0;0;2620;7,225;;C;6;;
-3;0;Lefebre, Master. Henry Forbes;male;;3;1;4133;25,4667;;S;;;
-3;0;Lefebre, Miss. Ida;female;;3;1;4133;25,4667;;S;;;
-3;0;Lefebre, Miss. Jeannie;female;;3;1;4133;25,4667;;S;;;
-3;0;Lefebre, Miss. Mathilde;female;;3;1;4133;25,4667;;S;;;
-3;0;Lefebre, Mrs. Frank (Frances);female;;0;4;4133;25,4667;;S;;;
-3;0;Leinonen, Mr. Antti Gustaf;male;32;0;0;STON/O 2. 3101292;7,925;;S;;;
-3;0;Lemberopolous, Mr. Peter L;male;34,5;0;0;2683;6,4375;;C;;196;
-3;0;Lennon, Miss. Mary;female;;1;0;370371;15,5;;Q;;;
-3;0;Lennon, Mr. Denis;male;;1;0;370371;15,5;;Q;;;
-3;0;Leonard, Mr. Lionel;male;36;0;0;LINE;0;;S;;;
-3;0;Lester, Mr. James;male;39;0;0;A/4 48871;24,15;;S;;;
-3;0;Lievens, Mr. Rene Aime;male;24;0;0;345781;9,5;;S;;;
-3;0;Lindahl, Miss. Agda Thorilda Viktoria;female;25;0;0;347071;7,775;;S;;;
-3;0;Lindblom, Miss. Augusta Charlotta;female;45;0;0;347073;7,75;;S;;;
-3;0;Lindell, Mr. Edvard Bengtsson;male;36;1;0;349910;15,55;;S;A;;
-3;0;Lindell, Mrs. Edvard Bengtsson (Elin Gerda Persson);female;30;1;0;349910;15,55;;S;A;;
-3;1;Lindqvist, Mr. Eino William;male;20;1;0;STON/O 2. 3101285;7,925;;S;15;;
-3;0;Linehan, Mr. Michael;male;;0;0;330971;7,8792;;Q;;;
-3;0;Ling, Mr. Lee;male;28;0;0;1601;56,4958;;S;;;
-3;0;Lithman, Mr. Simon;male;;0;0;S.O./P.P. 251;7,55;;S;;;
-3;0;Lobb, Mr. William Arthur;male;30;1;0;A/5. 3336;16,1;;S;;;
-3;0;Lobb, Mrs. William Arthur (Cordelia K Stanlick);female;26;1;0;A/5. 3336;16,1;;S;;;
-3;0;Lockyer, Mr. Edward;male;;0;0;1222;7,8792;;S;;153;
-3;0;"Lovell, Mr. John Hall (""Henry"")";male;20,5;0;0;A/5 21173;7,25;;S;;;
-3;1;Lulic, Mr. Nikola;male;27;0;0;315098;8,6625;;S;15;;
-3;0;Lundahl, Mr. Johan Svensson;male;51;0;0;347743;7,0542;;S;;;
-3;1;Lundin, Miss. Olga Elida;female;23;0;0;347469;7,8542;;S;10;;
-3;1;Lundstrom, Mr. Thure Edvin;male;32;0;0;350403;7,5792;;S;15;;
-3;0;Lyntakoff, Mr. Stanko;male;;0;0;349235;7,8958;;S;;;
-3;0;MacKay, Mr. George William;male;;0;0;C.A. 42795;7,55;;S;;;
-3;1;"Madigan, Miss. Margaret ""Maggie""";female;;0;0;370370;7,75;;Q;15;;
-3;1;Madsen, Mr. Fridtjof Arne;male;24;0;0;C 17369;7,1417;;S;13;;
-3;0;Maenpaa, Mr. Matti Alexanteri;male;22;0;0;STON/O 2. 3101275;7,125;;S;;;
-3;0;Mahon, Miss. Bridget Delia;female;;0;0;330924;7,8792;;Q;;;
-3;0;Mahon, Mr. John;male;;0;0;AQ/4 3130;7,75;;Q;;;
-3;0;Maisner, Mr. Simon;male;;0;0;A/S 2816;8,05;;S;;;
-3;0;Makinen, Mr. Kalle Edvard;male;29;0;0;STON/O 2. 3101268;7,925;;S;;;
-3;1;Mamee, Mr. Hanna;male;;0;0;2677;7,2292;;C;15;;
-3;0;Mangan, Miss. Mary;female;30,5;0;0;364850;7,75;;Q;;61;
-3;1;Mannion, Miss. Margareth;female;;0;0;36866;7,7375;;Q;16;;
-3;0;Mardirosian, Mr. Sarkis;male;;0;0;2655;7,2292;F E46;C;;;
-3;0;Markoff, Mr. Marin;male;35;0;0;349213;7,8958;;C;;;
-3;0;Markun, Mr. Johann;male;33;0;0;349257;7,8958;;S;;;
-3;1;Masselmani, Mrs. Fatima;female;;0;0;2649;7,225;;C;C;;
-3;0;Matinoff, Mr. Nicola;male;;0;0;349255;7,8958;;C;;;
-3;1;"McCarthy, Miss. Catherine ""Katie""";female;;0;0;383123;7,75;;Q;15 16;;
-3;1;McCormack, Mr. Thomas Joseph;male;;0;0;367228;7,75;;Q;;;
-3;1;McCoy, Miss. Agnes;female;;2;0;367226;23,25;;Q;16;;
-3;1;McCoy, Miss. Alicia;female;;2;0;367226;23,25;;Q;16;;
-3;1;McCoy, Mr. Bernard;male;;2;0;367226;23,25;;Q;16;;
-3;1;McDermott, Miss. Brigdet Delia;female;;0;0;330932;7,7875;;Q;13;;
-3;0;McEvoy, Mr. Michael;male;;0;0;36568;15,5;;Q;;;
-3;1;McGovern, Miss. Mary;female;;0;0;330931;7,8792;;Q;13;;
-3;1;"McGowan, Miss. Anna ""Annie""";female;15;0;0;330923;8,0292;;Q;;;
-3;0;McGowan, Miss. Katherine;female;35;0;0;9232;7,75;;Q;;;
-3;0;McMahon, Mr. Martin;male;;0;0;370372;7,75;;Q;;;
-3;0;McNamee, Mr. Neal;male;24;1;0;376566;16,1;;S;;;
-3;0;McNamee, Mrs. Neal (Eileen O'Leary);female;19;1;0;376566;16,1;;S;;53;
-3;0;McNeill, Miss. Bridget;female;;0;0;370368;7,75;;Q;;;
-3;0;Meanwell, Miss. (Marion Ogden);female;;0;0;SOTON/O.Q. 392087;8,05;;S;;;
-3;0;Meek, Mrs. Thomas (Annie Louise Rowley);female;;0;0;343095;8,05;;S;;;
-3;0;Meo, Mr. Alfonzo;male;55,5;0;0;A.5. 11206;8,05;;S;;201;
-3;0;Mernagh, Mr. Robert;male;;0;0;368703;7,75;;Q;;;
-3;1;Midtsjo, Mr. Karl Albert;male;21;0;0;345501;7,775;;S;15;;
-3;0;Miles, Mr. Frank;male;;0;0;359306;8,05;;S;;;
-3;0;Mineff, Mr. Ivan;male;24;0;0;349233;7,8958;;S;;;
-3;0;Minkoff, Mr. Lazar;male;21;0;0;349211;7,8958;;S;;;
-3;0;Mionoff, Mr. Stoytcho;male;28;0;0;349207;7,8958;;S;;;
-3;0;Mitkoff, Mr. Mito;male;;0;0;349221;7,8958;;S;;;
-3;1;"Mockler, Miss. Helen Mary ""Ellie""";female;;0;0;330980;7,8792;;Q;16;;
-3;0;Moen, Mr. Sigurd Hansen;male;25;0;0;348123;7,65;F G73;S;;309;
-3;1;Moor, Master. Meier;male;6;0;1;392096;12,475;E121;S;14;;
-3;1;Moor, Mrs. (Beila);female;27;0;1;392096;12,475;E121;S;14;;
-3;0;Moore, Mr. Leonard Charles;male;;0;0;A4. 54510;8,05;;S;;;
-3;1;Moran, Miss. Bertha;female;;1;0;371110;24,15;;Q;16;;
-3;0;Moran, Mr. Daniel J;male;;1;0;371110;24,15;;Q;;;
-3;0;Moran, Mr. James;male;;0;0;330877;8,4583;;Q;;;
-3;0;Morley, Mr. William;male;34;0;0;364506;8,05;;S;;;
-3;0;Morrow, Mr. Thomas Rowan;male;;0;0;372622;7,75;;Q;;;
-3;1;Moss, Mr. Albert Johan;male;;0;0;312991;7,775;;S;B;;
-3;1;Moubarek, Master. Gerios;male;;1;1;2661;15,2458;;C;C;;
-3;1;"Moubarek, Master. Halim Gonios (""William George"")";male;;1;1;2661;15,2458;;C;C;;
-3;1;"Moubarek, Mrs. George (Omine ""Amenia"" Alexander)";female;;0;2;2661;15,2458;;C;C;;
-3;1;Moussa, Mrs. (Mantoura Boulos);female;;0;0;2626;7,2292;;C;;;
-3;0;Moutal, Mr. Rahamin Haim;male;;0;0;374746;8,05;;S;;;
-3;1;"Mullens, Miss. Katherine ""Katie""";female;;0;0;35852;7,7333;;Q;16;;
-3;1;Mulvihill, Miss. Bertha E;female;24;0;0;382653;7,75;;Q;15;;
-3;0;Murdlin, Mr. Joseph;male;;0;0;A./5. 3235;8,05;;S;;;
-3;1;"Murphy, Miss. Katherine ""Kate""";female;;1;0;367230;15,5;;Q;16;;
-3;1;Murphy, Miss. Margaret Jane;female;;1;0;367230;15,5;;Q;16;;
-3;1;Murphy, Miss. Nora;female;;0;0;36568;15,5;;Q;16;;
-3;0;Myhrman, Mr. Pehr Fabian Oliver Malkolm;male;18;0;0;347078;7,75;;S;;;
-3;0;Naidenoff, Mr. Penko;male;22;0;0;349206;7,8958;;S;;;
-3;1;"Najib, Miss. Adele Kiamie ""Jane""";female;15;0;0;2667;7,225;;C;C;;
-3;1;"Nakid, Miss. Maria (""Mary"")";female;1;0;2;2653;15,7417;;C;C;;
-3;1;Nakid, Mr. Sahid;male;20;1;1;2653;15,7417;;C;C;;
-3;1;"Nakid, Mrs. Said (Waika ""Mary"" Mowad)";female;19;1;1;2653;15,7417;;C;C;;
-3;0;Nancarrow, Mr. William Henry;male;33;0;0;A./5. 3338;8,05;;S;;;
-3;0;Nankoff, Mr. Minko;male;;0;0;349218;7,8958;;S;;;
-3;0;Nasr, Mr. Mustafa;male;;0;0;2652;7,2292;;C;;;
-3;0;Naughton, Miss. Hannah;female;;0;0;365237;7,75;;Q;;;
-3;0;Nenkoff, Mr. Christo;male;;0;0;349234;7,8958;;S;;;
-3;1;Nicola-Yarred, Master. Elias;male;12;1;0;2651;11,2417;;C;C;;
-3;1;Nicola-Yarred, Miss. Jamila;female;14;1;0;2651;11,2417;;C;C;;
-3;0;Nieminen, Miss. Manta Josefina;female;29;0;0;3101297;7,925;;S;;;
-3;0;Niklasson, Mr. Samuel;male;28;0;0;363611;8,05;;S;;;
-3;1;Nilsson, Miss. Berta Olivia;female;18;0;0;347066;7,775;;S;D;;
-3;1;Nilsson, Miss. Helmina Josefina;female;26;0;0;347470;7,8542;;S;13;;
-3;0;Nilsson, Mr. August Ferdinand;male;21;0;0;350410;7,8542;;S;;;
-3;0;Nirva, Mr. Iisakki Antino Aijo;male;41;0;0;SOTON/O2 3101272;7,125;;S;;;Finland Sudbury, ON
-3;1;Niskanen, Mr. Juha;male;39;0;0;STON/O 2. 3101289;7,925;;S;9;;
-3;0;Nosworthy, Mr. Richard Cater;male;21;0;0;A/4. 39886;7,8;;S;;;
-3;0;Novel, Mr. Mansouer;male;28,5;0;0;2697;7,2292;;C;;181;
-3;1;Nysten, Miss. Anna Sofia;female;22;0;0;347081;7,75;;S;13;;
-3;0;Nysveen, Mr. Johan Hansen;male;61;0;0;345364;6,2375;;S;;;
-3;0;O'Brien, Mr. Thomas;male;;1;0;370365;15,5;;Q;;;
-3;0;O'Brien, Mr. Timothy;male;;0;0;330979;7,8292;;Q;;;
-3;1;"O'Brien, Mrs. Thomas (Johanna ""Hannah"" Godfrey)";female;;1;0;370365;15,5;;Q;;;
-3;0;O'Connell, Mr. Patrick D;male;;0;0;334912;7,7333;;Q;;;
-3;0;O'Connor, Mr. Maurice;male;;0;0;371060;7,75;;Q;;;
-3;0;O'Connor, Mr. Patrick;male;;0;0;366713;7,75;;Q;;;
-3;0;Odahl, Mr. Nils Martin;male;23;0;0;7267;9,225;;S;;;
-3;0;O'Donoghue, Ms. Bridget;female;;0;0;364856;7,75;;Q;;;
-3;1;O'Driscoll, Miss. Bridget;female;;0;0;14311;7,75;;Q;D;;
-3;1;"O'Dwyer, Miss. Ellen ""Nellie""";female;;0;0;330959;7,8792;;Q;;;
-3;1;Ohman, Miss. Velin;female;22;0;0;347085;7,775;;S;C;;
-3;1;O'Keefe, Mr. Patrick;male;;0;0;368402;7,75;;Q;B;;
-3;1;"O'Leary, Miss. Hanora ""Norah""";female;;0;0;330919;7,8292;;Q;13;;
-3;1;Olsen, Master. Artur Karl;male;9;0;1;C 17368;3,1708;;S;13;;
-3;0;Olsen, Mr. Henry Margido;male;28;0;0;C 4001;22,525;;S;;173;
-3;0;Olsen, Mr. Karl Siegwart Andreas;male;42;0;1;4579;8,4042;;S;;;
-3;0;Olsen, Mr. Ole Martin;male;;0;0;Fa 265302;7,3125;;S;;;
-3;0;Olsson, Miss. Elina;female;31;0;0;350407;7,8542;;S;;;
-3;0;Olsson, Mr. Nils Johan Goransson;male;28;0;0;347464;7,8542;;S;;;
-3;1;Olsson, Mr. Oscar Wilhelm;male;32;0;0;347079;7,775;;S;A;;
-3;0;Olsvigen, Mr. Thor Anderson;male;20;0;0;6563;9,225;;S;;89;Oslo, Norway Cameron, WI
-3;0;Oreskovic, Miss. Jelka;female;23;0;0;315085;8,6625;;S;;;
-3;0;Oreskovic, Miss. Marija;female;20;0;0;315096;8,6625;;S;;;
-3;0;Oreskovic, Mr. Luka;male;20;0;0;315094;8,6625;;S;;;
-3;0;Osen, Mr. Olaf Elon;male;16;0;0;7534;9,2167;;S;;;
-3;1;Osman, Mrs. Mara;female;31;0;0;349244;8,6833;;S;;;
-3;0;O'Sullivan, Miss. Bridget Mary;female;;0;0;330909;7,6292;;Q;;;
-3;0;Palsson, Master. Gosta Leonard;male;2;3;1;349909;21,075;;S;;4;
-3;0;Palsson, Master. Paul Folke;male;6;3;1;349909;21,075;;S;;;
-3;0;Palsson, Miss. Stina Viola;female;3;3;1;349909;21,075;;S;;;
-3;0;Palsson, Miss. Torborg Danira;female;8;3;1;349909;21,075;;S;;;
-3;0;Palsson, Mrs. Nils (Alma Cornelia Berglund);female;29;0;4;349909;21,075;;S;;206;
-3;0;Panula, Master. Eino Viljami;male;1;4;1;3101295;39,6875;;S;;;
-3;0;Panula, Master. Juha Niilo;male;7;4;1;3101295;39,6875;;S;;;
-3;0;Panula, Master. Urho Abraham;male;2;4;1;3101295;39,6875;;S;;;
-3;0;Panula, Mr. Ernesti Arvid;male;16;4;1;3101295;39,6875;;S;;;
-3;0;Panula, Mr. Jaako Arnold;male;14;4;1;3101295;39,6875;;S;;;
-3;0;Panula, Mrs. Juha (Maria Emilia Ojala);female;41;0;5;3101295;39,6875;;S;;;
-3;0;Pasic, Mr. Jakob;male;21;0;0;315097;8,6625;;S;;;
-3;0;Patchett, Mr. George;male;19;0;0;358585;14,5;;S;;;
-3;0;Paulner, Mr. Uscher;male;;0;0;3411;8,7125;;C;;;
-3;0;Pavlovic, Mr. Stefo;male;32;0;0;349242;7,8958;;S;;;
-3;0;Peacock, Master. Alfred Edward;male;0,75;1;1;SOTON/O.Q. 3101315;13,775;;S;;;
-3;0;Peacock, Miss. Treasteall;female;3;1;1;SOTON/O.Q. 3101315;13,775;;S;;;
-3;0;Peacock, Mrs. Benjamin (Edith Nile);female;26;0;2;SOTON/O.Q. 3101315;13,775;;S;;;
-3;0;Pearce, Mr. Ernest;male;;0;0;343271;7;;S;;;
-3;0;Pedersen, Mr. Olaf;male;;0;0;345498;7,775;;S;;;
-3;0;Peduzzi, Mr. Joseph;male;;0;0;A/5 2817;8,05;;S;;;
-3;0;Pekoniemi, Mr. Edvard;male;21;0;0;STON/O 2. 3101294;7,925;;S;;;
-3;0;Peltomaki, Mr. Nikolai Johannes;male;25;0;0;STON/O 2. 3101291;7,925;;S;;;
-3;0;Perkin, Mr. John Henry;male;22;0;0;A/5 21174;7,25;;S;;;
-3;1;Persson, Mr. Ernst Ulrik;male;25;1;0;347083;7,775;;S;15;;
-3;1;Peter, Master. Michael J;male;;1;1;2668;22,3583;;C;C;;
-3;1;Peter, Miss. Anna;female;;1;1;2668;22,3583;F E69;C;D;;
-3;1;Peter, Mrs. Catherine (Catherine Rizk);female;;0;2;2668;22,3583;;C;D;;
-3;0;Peters, Miss. Katie;female;;0;0;330935;8,1375;;Q;;;
-3;0;Petersen, Mr. Marius;male;24;0;0;342441;8,05;;S;;;
-3;0;Petranec, Miss. Matilda;female;28;0;0;349245;7,8958;;S;;;
-3;0;Petroff, Mr. Nedelio;male;19;0;0;349212;7,8958;;S;;;
-3;0;"Petroff, Mr. Pastcho (""Pentcho"")";male;;0;0;349215;7,8958;;S;;;
-3;0;Petterson, Mr. Johan Emil;male;25;1;0;347076;7,775;;S;;;
-3;0;Pettersson, Miss. Ellen Natalia;female;18;0;0;347087;7,775;;S;;;
-3;1;Pickard, Mr. Berk (Berk Trembisky);male;32;0;0;SOTON/O.Q. 392078;8,05;E10;S;9;;
-3;0;Plotcharsky, Mr. Vasil;male;;0;0;349227;7,8958;;S;;;
-3;0;Pokrnic, Mr. Mate;male;17;0;0;315095;8,6625;;S;;;
-3;0;Pokrnic, Mr. Tome;male;24;0;0;315092;8,6625;;S;;;
-3;0;Radeff, Mr. Alexander;male;;0;0;349223;7,8958;;S;;;
-3;0;Rasmussen, Mrs. (Lena Jacobsen Solvang);female;;0;0;65305;8,1125;;S;;;
-3;0;Razi, Mr. Raihed;male;;0;0;2629;7,2292;;C;;;
-3;0;Reed, Mr. James George;male;;0;0;362316;7,25;;S;;;
-3;0;Rekic, Mr. Tido;male;38;0;0;349249;7,8958;;S;;;
-3;0;Reynolds, Mr. Harold J;male;21;0;0;342684;8,05;;S;;;
-3;0;Rice, Master. Albert;male;10;4;1;382652;29,125;;Q;;;
-3;0;Rice, Master. Arthur;male;4;4;1;382652;29,125;;Q;;;
-3;0;Rice, Master. Eric;male;7;4;1;382652;29,125;;Q;;;
-3;0;Rice, Master. Eugene;male;2;4;1;382652;29,125;;Q;;;
-3;0;Rice, Master. George Hugh;male;8;4;1;382652;29,125;;Q;;;
-3;0;Rice, Mrs. William (Margaret Norton);female;39;0;5;382652;29,125;;Q;;327;
-3;0;"Riihivouri, Miss. Susanna Juhantytar ""Sanni""";female;22;0;0;3101295;39,6875;;S;;;
-3;0;Rintamaki, Mr. Matti;male;35;0;0;STON/O 2. 3101273;7,125;;S;;;
-3;1;"Riordan, Miss. Johanna ""Hannah""";female;;0;0;334915;7,7208;;Q;13;;
-3;0;Risien, Mr. Samuel Beard;male;;0;0;364498;14,5;;S;;;
-3;0;Risien, Mrs. Samuel (Emma);female;;0;0;364498;14,5;;S;;;
-3;0;Robins, Mr. Alexander A;male;50;1;0;A/5. 3337;14,5;;S;;119;
-3;0;Robins, Mrs. Alexander A (Grace Charity Laury);female;47;1;0;A/5. 3337;14,5;;S;;7;
-3;0;Rogers, Mr. William John;male;;0;0;S.C./A.4. 23567;8,05;;S;;;
-3;0;Rommetvedt, Mr. Knud Paust;male;;0;0;312993;7,775;;S;;;
-3;0;Rosblom, Miss. Salli Helena;female;2;1;1;370129;20,2125;;S;;;
-3;0;Rosblom, Mr. Viktor Richard;male;18;1;1;370129;20,2125;;S;;;
-3;0;Rosblom, Mrs. Viktor (Helena Wilhelmina);female;41;0;2;370129;20,2125;;S;;;
-3;1;Roth, Miss. Sarah A;female;;0;0;342712;8,05;;S;C;;
-3;0;Rouse, Mr. Richard Henry;male;50;0;0;A/5 3594;8,05;;S;;;
-3;0;Rush, Mr. Alfred George John;male;16;0;0;A/4. 20589;8,05;;S;;;
-3;1;Ryan, Mr. Edward;male;;0;0;383162;7,75;;Q;14;;
-3;0;Ryan, Mr. Patrick;male;;0;0;371110;24,15;;Q;;;
-3;0;Saad, Mr. Amin;male;;0;0;2671;7,2292;;C;;;
-3;0;Saad, Mr. Khalil;male;25;0;0;2672;7,225;;C;;;
-3;0;Saade, Mr. Jean Nassr;male;;0;0;2676;7,225;;C;;;
-3;0;Sadlier, Mr. Matthew;male;;0;0;367655;7,7292;;Q;;;
-3;0;Sadowitz, Mr. Harry;male;;0;0;LP 1588;7,575;;S;;;
-3;0;Saether, Mr. Simon Sivertsen;male;38,5;0;0;SOTON/O.Q. 3101262;7,25;;S;;32;
-3;0;Sage, Master. Thomas Henry;male;;8;2;CA. 2343;69,55;;S;;;
-3;0;Sage, Master. William Henry;male;14,5;8;2;CA. 2343;69,55;;S;;67;
-3;0;Sage, Miss. Ada;female;;8;2;CA. 2343;69,55;;S;;;
-3;0;Sage, Miss. Constance Gladys;female;;8;2;CA. 2343;69,55;;S;;;
-3;0;"Sage, Miss. Dorothy Edith ""Dolly""";female;;8;2;CA. 2343;69,55;;S;;;
-3;0;Sage, Miss. Stella Anna;female;;8;2;CA. 2343;69,55;;S;;;
-3;0;Sage, Mr. Douglas Bullen;male;;8;2;CA. 2343;69,55;;S;;;
-3;0;Sage, Mr. Frederick;male;;8;2;CA. 2343;69,55;;S;;;
-3;0;Sage, Mr. George John Jr;male;;8;2;CA. 2343;69,55;;S;;;
-3;0;Sage, Mr. John George;male;;1;9;CA. 2343;69,55;;S;;;
-3;0;Sage, Mrs. John (Annie Bullen);female;;1;9;CA. 2343;69,55;;S;;;
-3;0;Salander, Mr. Karl Johan;male;24;0;0;7266;9,325;;S;;;
-3;1;Salkjelsvik, Miss. Anna Kristine;female;21;0;0;343120;7,65;;S;C;;
-3;0;Salonen, Mr. Johan Werner;male;39;0;0;3101296;7,925;;S;;;
-3;0;Samaan, Mr. Elias;male;;2;0;2662;21,6792;;C;;;
-3;0;Samaan, Mr. Hanna;male;;2;0;2662;21,6792;;C;;;
-3;0;Samaan, Mr. Youssef;male;;2;0;2662;21,6792;;C;;;
-3;1;Sandstrom, Miss. Beatrice Irene;female;1;1;1;PP 9549;16,7;G6;S;13;;
-3;1;Sandstrom, Mrs. Hjalmar (Agnes Charlotta Bengtsson);female;24;0;2;PP 9549;16,7;G6;S;13;;
-3;1;Sandstrom, Miss. Marguerite Rut;female;4;1;1;PP 9549;16,7;G6;S;13;;
-3;1;Sap, Mr. Julius;male;25;0;0;345768;9,5;;S;11;;
-3;0;Saundercock, Mr. William Henry;male;20;0;0;A/5. 2151;8,05;;S;;;
-3;0;Sawyer, Mr. Frederick Charles;male;24,5;0;0;342826;8,05;;S;;284;
-3;0;Scanlan, Mr. James;male;;0;0;36209;7,725;;Q;;;
-3;0;Sdycoff, Mr. Todor;male;;0;0;349222;7,8958;;S;;;
-3;0;Shaughnessy, Mr. Patrick;male;;0;0;370374;7,75;;Q;;;
-3;1;Sheerlinck, Mr. Jan Baptist;male;29;0;0;345779;9,5;;S;11;;
-3;0;Shellard, Mr. Frederick William;male;;0;0;C.A. 6212;15,1;;S;;;
-3;1;Shine, Miss. Ellen Natalia;female;;0;0;330968;7,7792;;Q;;;
-3;0;Shorney, Mr. Charles Joseph;male;;0;0;374910;8,05;;S;;;
-3;0;Simmons, Mr. John;male;;0;0;SOTON/OQ 392082;8,05;;S;;;
-3;0;Sirayanian, Mr. Orsen;male;22;0;0;2669;7,2292;;C;;;
-3;0;Sirota, Mr. Maurice;male;;0;0;392092;8,05;;S;;;
-3;0;Sivic, Mr. Husein;male;40;0;0;349251;7,8958;;S;;;
-3;0;Sivola, Mr. Antti Wilhelm;male;21;0;0;STON/O 2. 3101280;7,925;;S;;;
-3;1;Sjoblom, Miss. Anna Sofia;female;18;0;0;3101265;7,4958;;S;16;;
-3;0;Skoog, Master. Harald;male;4;3;2;347088;27,9;;S;;;
-3;0;Skoog, Master. Karl Thorsten;male;10;3;2;347088;27,9;;S;;;
-3;0;Skoog, Miss. Mabel;female;9;3;2;347088;27,9;;S;;;
-3;0;Skoog, Miss. Margit Elizabeth;female;2;3;2;347088;27,9;;S;;;
-3;0;Skoog, Mr. Wilhelm;male;40;1;4;347088;27,9;;S;;;
-3;0;Skoog, Mrs. William (Anna Bernhardina Karlsson);female;45;1;4;347088;27,9;;S;;;
-3;0;Slabenoff, Mr. Petco;male;;0;0;349214;7,8958;;S;;;
-3;0;Slocovski, Mr. Selman Francis;male;;0;0;SOTON/OQ 392086;8,05;;S;;;
-3;0;Smiljanic, Mr. Mile;male;;0;0;315037;8,6625;;S;;;
-3;0;Smith, Mr. Thomas;male;;0;0;384461;7,75;;Q;;;
-3;1;Smyth, Miss. Julia;female;;0;0;335432;7,7333;;Q;13;;
-3;0;Soholt, Mr. Peter Andreas Lauritz Andersen;male;19;0;0;348124;7,65;F G73;S;;;
-3;0;Somerton, Mr. Francis William;male;30;0;0;A.5. 18509;8,05;;S;;;
-3;0;Spector, Mr. Woolf;male;;0;0;A.5. 3236;8,05;;S;;;
-3;0;Spinner, Mr. Henry John;male;32;0;0;STON/OQ. 369943;8,05;;S;;;
-3;0;Staneff, Mr. Ivan;male;;0;0;349208;7,8958;;S;;;
-3;0;Stankovic, Mr. Ivan;male;33;0;0;349239;8,6625;;C;;;
-3;1;Stanley, Miss. Amy Zillah Elsie;female;23;0;0;CA. 2314;7,55;;S;C;;
-3;0;Stanley, Mr. Edward Roland;male;21;0;0;A/4 45380;8,05;;S;;;
-3;0;Storey, Mr. Thomas;male;60,5;0;0;3701;;;S;;261;
-3;0;Stoytcheff, Mr. Ilia;male;19;0;0;349205;7,8958;;S;;;
-3;0;Strandberg, Miss. Ida Sofia;female;22;0;0;7553;9,8375;;S;;;
-3;1;Stranden, Mr. Juho;male;31;0;0;STON/O 2. 3101288;7,925;;S;9;;
-3;0;Strilic, Mr. Ivan;male;27;0;0;315083;8,6625;;S;;;
-3;0;Strom, Miss. Telma Matilda;female;2;0;1;347054;10,4625;G6;S;;;
-3;0;Strom, Mrs. Wilhelm (Elna Matilda Persson);female;29;1;1;347054;10,4625;G6;S;;;
-3;1;Sunderland, Mr. Victor Francis;male;16;0;0;SOTON/OQ 392089;8,05;;S;B;;
-3;1;Sundman, Mr. Johan Julian;male;44;0;0;STON/O 2. 3101269;7,925;;S;15;;
-3;0;Sutehall, Mr. Henry Jr;male;25;0;0;SOTON/OQ 392076;7,05;;S;;;
-3;0;Svensson, Mr. Johan;male;74;0;0;347060;7,775;;S;;;
-3;1;Svensson, Mr. Johan Cervin;male;14;0;0;7538;9,225;;S;13;;
-3;0;Svensson, Mr. Olof;male;24;0;0;350035;7,7958;;S;;;
-3;1;Tenglin, Mr. Gunnar Isidor;male;25;0;0;350033;7,7958;;S;13 15;;
-3;0;Theobald, Mr. Thomas Leonard;male;34;0;0;363294;8,05;;S;;176;
-3;1;Thomas, Master. Assad Alexander;male;0,4167;0;1;2625;8,5167;;C;16;;
-3;0;Thomas, Mr. Charles P;male;;1;0;2621;6,4375;;C;;;
-3;0;Thomas, Mr. John;male;;0;0;2681;6,4375;;C;;;
-3;0;Thomas, Mr. Tannous;male;;0;0;2684;7,225;;C;;;
-3;1;"Thomas, Mrs. Alexander (Thamine ""Thelma"")";female;16;1;1;2625;8,5167;;C;14;;
-3;0;Thomson, Mr. Alexander Morrison;male;;0;0;32302;8,05;;S;;;
-3;0;Thorneycroft, Mr. Percival;male;;1;0;376564;16,1;;S;;;
-3;1;Thorneycroft, Mrs. Percival (Florence Kate White);female;;1;0;376564;16,1;;S;10;;
-3;0;Tikkanen, Mr. Juho;male;32;0;0;STON/O 2. 3101293;7,925;;S;;;
-3;0;Tobin, Mr. Roger;male;;0;0;383121;7,75;F38;Q;;;
-3;0;Todoroff, Mr. Lalio;male;;0;0;349216;7,8958;;S;;;
-3;0;Tomlin, Mr. Ernest Portage;male;30,5;0;0;364499;8,05;;S;;50;
-3;0;Torber, Mr. Ernst William;male;44;0;0;364511;8,05;;S;;;
-3;0;Torfa, Mr. Assad;male;;0;0;2673;7,2292;;C;;;
-3;1;Tornquist, Mr. William Henry;male;25;0;0;LINE;0;;S;15;;
-3;0;Toufik, Mr. Nakli;male;;0;0;2641;7,2292;;C;;;
-3;1;Touma, Master. Georges Youssef;male;7;1;1;2650;15,2458;;C;C;;
-3;1;Touma, Miss. Maria Youssef;female;9;1;1;2650;15,2458;;C;C;;
-3;1;Touma, Mrs. Darwis (Hanne Youssef Razi);female;29;0;2;2650;15,2458;;C;C;;
-3;0;Turcin, Mr. Stjepan;male;36;0;0;349247;7,8958;;S;;;
-3;1;Turja, Miss. Anna Sofia;female;18;0;0;4138;9,8417;;S;15;;
-3;1;Turkula, Mrs. (Hedwig);female;63;0;0;4134;9,5875;;S;15;;
-3;0;van Billiard, Master. James William;male;;1;1;A/5. 851;14,5;;S;;;
-3;0;van Billiard, Master. Walter John;male;11,5;1;1;A/5. 851;14,5;;S;;1;
-3;0;van Billiard, Mr. Austin Blyler;male;40,5;0;2;A/5. 851;14,5;;S;;255;
-3;0;Van Impe, Miss. Catharina;female;10;0;2;345773;24,15;;S;;;
-3;0;Van Impe, Mr. Jean Baptiste;male;36;1;1;345773;24,15;;S;;;
-3;0;Van Impe, Mrs. Jean Baptiste (Rosalie Paula Govaert);female;30;1;1;345773;24,15;;S;;;
-3;0;van Melkebeke, Mr. Philemon;male;;0;0;345777;9,5;;S;;;
-3;0;Vande Velde, Mr. Johannes Joseph;male;33;0;0;345780;9,5;;S;;;
-3;0;Vande Walle, Mr. Nestor Cyriel;male;28;0;0;345770;9,5;;S;;;
-3;0;Vanden Steen, Mr. Leo Peter;male;28;0;0;345783;9,5;;S;;;
-3;0;Vander Cruyssen, Mr. Victor;male;47;0;0;345765;9;;S;;;
-3;0;Vander Planke, Miss. Augusta Maria;female;18;2;0;345764;18;;S;;;
-3;0;Vander Planke, Mr. Julius;male;31;3;0;345763;18;;S;;;
-3;0;Vander Planke, Mr. Leo Edmondus;male;16;2;0;345764;18;;S;;;
-3;0;Vander Planke, Mrs. Julius (Emelia Maria Vandemoortele);female;31;1;0;345763;18;;S;;;
-3;1;Vartanian, Mr. David;male;22;0;0;2658;7,225;;C;13 15;;
-3;0;Vendel, Mr. Olof Edvin;male;20;0;0;350416;7,8542;;S;;;
-3;0;Vestrom, Miss. Hulda Amanda Adolfina;female;14;0;0;350406;7,8542;;S;;;
-3;0;Vovk, Mr. Janko;male;22;0;0;349252;7,8958;;S;;;
-3;0;Waelens, Mr. Achille;male;22;0;0;345767;9;;S;;;Antwerp, Belgium / Stanton, OH
-3;0;Ware, Mr. Frederick;male;;0;0;359309;8,05;;S;;;
-3;0;Warren, Mr. Charles William;male;;0;0;C.A. 49867;7,55;;S;;;
-3;0;Webber, Mr. James;male;;0;0;SOTON/OQ 3101316;8,05;;S;;;
-3;0;Wenzel, Mr. Linhart;male;32,5;0;0;345775;9,5;;S;;298;
-3;1;Whabee, Mrs. George Joseph (Shawneene Abi-Saab);female;38;0;0;2688;7,2292;;C;C;;
-3;0;Widegren, Mr. Carl/Charles Peter;male;51;0;0;347064;7,75;;S;;;
-3;0;Wiklund, Mr. Jakob Alfred;male;18;1;0;3101267;6,4958;;S;;314;
-3;0;Wiklund, Mr. Karl Johan;male;21;1;0;3101266;6,4958;;S;;;
-3;1;Wilkes, Mrs. James (Ellen Needs);female;47;1;0;363272;7;;S;;;
-3;0;"Willer, Mr. Aaron (""Abi Weller"")";male;;0;0;3410;8,7125;;S;;;
-3;0;Willey, Mr. Edward;male;;0;0;S.O./P.P. 751;7,55;;S;;;
-3;0;"Williams, Mr. Howard Hugh ""Harry""";male;;0;0;A/5 2466;8,05;;S;;;
-3;0;Williams, Mr. Leslie;male;28,5;0;0;54636;16,1;;S;;14;
-3;0;Windelov, Mr. Einar;male;21;0;0;SOTON/OQ 3101317;7,25;;S;;;
-3;0;Wirz, Mr. Albert;male;27;0;0;315154;8,6625;;S;;131;
-3;0;Wiseman, Mr. Phillippe;male;;0;0;A/4. 34244;7,25;;S;;;
-3;0;Wittevrongel, Mr. Camille;male;36;0;0;345771;9,5;;S;;;
-3;0;Yasbeck, Mr. Antoni;male;27;1;0;2659;14,4542;;C;C;;
-3;1;Yasbeck, Mrs. Antoni (Selini Alexander);female;15;1;0;2659;14,4542;;C;;;
-3;0;Youseff, Mr. Gerious;male;45,5;0;0;2628;7,225;;C;;312;
-3;0;Yousif, Mr. Wazli;male;;0;0;2647;7,225;;C;;;
-3;0;Yousseff, Mr. Gerious;male;;0;0;2627;14,4583;;C;;;
-3;0;Zabour, Miss. Hileni;female;14,5;1;0;2665;14,4542;;C;;328;
-3;0;Zabour, Miss. Thamine;female;;1;0;2665;14,4542;;C;;;
-3;0;Zakarian, Mr. Mapriededer;male;26,5;0;0;2656;7,225;;C;;304;
-3;0;Zakarian, Mr. Ortin;male;27;0;0;2670;7,225;;C;;;
-3;0;Zimmerman, Mr. Leo;male;29;0;0;315082;7,875;;S;;;
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
index c82d5de..b613ade 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
@@ -22,7 +22,6 @@
import org.apache.ignite.ml.composition.CompositionTestSuite;
import org.apache.ignite.ml.dataset.DatasetTestSuite;
import org.apache.ignite.ml.environment.EnvironmentTestSuite;
-import org.apache.ignite.ml.genetic.GAGridTestSuite;
import org.apache.ignite.ml.inference.InferenceTestSuite;
import org.apache.ignite.ml.knn.KNNTestSuite;
import org.apache.ignite.ml.math.MathImplMainTestSuite;
@@ -55,7 +54,6 @@
KNNTestSuite.class,
PipelineTestSuite.class,
PreprocessingTestSuite.class,
- GAGridTestSuite.class,
CompositionTestSuite.class,
EnvironmentTestSuite.class,
StructuresTestSuite.class,
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/genetic/GAGridCalculateFitnessTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/genetic/GAGridCalculateFitnessTest.java
deleted file mode 100644
index fa05bad..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/genetic/GAGridCalculateFitnessTest.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.SqlFieldsQuery;
-import org.apache.ignite.ml.genetic.parameter.GAConfiguration;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Calculate Fitness Test
- */
-public class GAGridCalculateFitnessTest {
- /** Ignite instance */
- private Ignite ignite;
-
- /** GAGrid **/
- private GAGrid gaGrid;
-
- /** GAConfiguration */
- private GAConfiguration gaCfg;
-
- /**
- * Setup test
- */
- @Before
- public void initialize() {
-
- try {
-
- // Create an Ignite instance as you would in any other use case.
- ignite = Ignition.start();
-
- // Create GAConfiguration
- gaCfg = new GAConfiguration();
-
- // set Gene Pool
- List<Gene> genes = this.getGenePool();
- gaCfg.setGenePool(genes);
-
- // set the Chromosome Length to '8' since password contains 8 characters.
- gaCfg.setChromosomeLen(8);
-
- // create and set Fitness function
- PasswordFitnessFunction function = new PasswordFitnessFunction();
- gaCfg.setFitnessFunction(function);
-
- gaGrid = new GAGrid(gaCfg, ignite);
- gaGrid.initializeGenePopulation();
- gaGrid.initializePopulation();
-
- }
- catch (Exception e) {
- System.out.println(e.getMessage());
- }
- }
-
- /**
- * Test Calculate Fitness
- */
- @Test
- public void testCalculateFitness() {
- try {
-
- List<Long> chromosomeKeys = gaGrid.getPopulationKeys();
-
- Boolean boolVal = this.ignite.compute().execute(new FitnessTask(this.gaCfg), chromosomeKeys);
-
- IgniteCache<Long, Chromosome> populationCache = ignite.cache(GAGridConstants.POPULATION_CACHE);
-
- String sql = "select count(*) from Chromosome where fitnessScore>0";
-
- // Execute query to keys for ALL Chromosomes by fitnessScore
- QueryCursor<List<?>> cursor = populationCache.query(new SqlFieldsQuery(sql));
-
- List<List<?>> res = cursor.getAll();
-
- Long cnt = 0L;
-
- for (List row : res)
- cnt = (Long)row.get(0);
-
- assertEquals(500, cnt.longValue());
- }
-
- catch (Exception e) {
- System.out.println(e.getMessage());
- }
- }
-
- /**
- * Helper routine to initialize Gene pool
- *
- * @return List of Genes
- */
- private List<Gene> getGenePool() {
- List<Gene> list = new ArrayList();
-
- char[] chars = {
- 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's',
- 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S',
- 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2', '3', '4', '5', '6', '7', '8', '!', '"', '#', '$', '%', '&', '(', ')', '*', '+', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', ']', '^'};
-
- for (int i = 0; i < chars.length; i++) {
- Gene gene = new Gene(chars[i]);
- list.add(gene);
- }
- return list;
- }
-
- /**
- * Stop the Ignite.
- */
- @After
- public void tearDown() {
-
- Ignition.stop(true);
- ignite = null;
- }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/genetic/GAGridInitializePopulationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/genetic/GAGridInitializePopulationTest.java
deleted file mode 100644
index 1fd0538..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/genetic/GAGridInitializePopulationTest.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.SqlFieldsQuery;
-import org.apache.ignite.ml.genetic.parameter.GAConfiguration;
-import org.apache.ignite.ml.genetic.parameter.GAGridConstants;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Initialize Gene and Chromosome Test
- */
-public class GAGridInitializePopulationTest {
- /** Ignite instance */
- private Ignite ignite;
-
- /** GAGrid **/
- private GAGrid gaGrid;
-
- /**
- * Initialize the population.
- */
- @Before
- public void initialize() {
- try {
-
- // Create an Ignite instance as you would in any other use case.
- ignite = Ignition.start();
-
- // Create GAConfiguration
- /* GAConfiguraton */ /** GAConfiguraton */GAConfiguration gaCfg = new GAConfiguration();
-
- // set Gene Pool
- List<Gene> genes = this.getGenePool();
-
- // set the Chromosome Length to '8' since password contains 8 characters.
- gaCfg.setChromosomeLen(8);
-
- gaCfg.setGenePool(genes);
-
- gaGrid = new GAGrid(gaCfg, ignite);
- }
- catch (Exception e) {
- System.out.println(e.getMessage());
- }
- }
-
- /**
- * Check the initialization of genes.
- */
- @Test
- public void testInitializeGenes() {
- try {
- IgniteCache<Long, Gene> geneCache = ignite.cache(GAGridConstants.GENE_CACHE);
- gaGrid.initializeGenePopulation();
-
- String sql = "select count(*) from Gene";
-
- // Execute query to keys for ALL Chromosomes by fittnessScore
- QueryCursor<List<?>> cursor = geneCache.query(new SqlFieldsQuery(sql));
-
- List<List<?>> res = cursor.getAll();
-
- Long cnt = 0L;
-
- for (List row : res)
- cnt = (Long)row.get(0);
-
- assertEquals(83, cnt.longValue());
- }
-
- catch (Exception e) {
- System.out.println(e.getMessage());
- }
- }
-
- /**
- *
- */
- @Test
- public void testInitializePopulation() {
- try {
-
- IgniteCache<Long, Chromosome> populationCache = ignite.cache(GAGridConstants.POPULATION_CACHE);
-
- gaGrid.initializePopulation();
-
- String sql = "select count(*) from Chromosome";
-
- // Execute query to keys for ALL Chromosomes by fittnessScore
- QueryCursor<List<?>> cursor = populationCache.query(new SqlFieldsQuery(sql));
-
- List<List<?>> res = cursor.getAll();
-
- Long cnt = 0L;
-
- for (List row : res)
- cnt = (Long)row.get(0);
-
- assertEquals(500, cnt.longValue());
- }
-
- catch (Exception e) {
- System.out.println(e.getMessage());
- }
- }
-
- /**
- * Helper routine to initialize Gene pool
- *
- * @return List of Genes
- */
- private List<Gene> getGenePool() {
- List<Gene> list = new ArrayList();
-
- char[] chars = {
- 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's',
- 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S',
- 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2', '3', '4', '5', '6', '7', '8', '!', '"', '#', '$', '%', '&', '(', ')', '*', '+', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', ']', '^'};
-
- for (int i = 0; i < chars.length; i++) {
- Gene gene = new Gene(chars[i]);
- list.add(gene);
- }
- return list;
- }
-
- /**
- * Stop the Ignite.
- */
- @After
- public void tearDown() {
-
- Ignition.stop(true);
- ignite = null;
- }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/genetic/GAGridTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/genetic/GAGridTestSuite.java
deleted file mode 100644
index d7751fd..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/genetic/GAGridTestSuite.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-
-/**
- * Test suite for all tests located in org.apache.ignite.ml.genetic package.
- */
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
- GAGridInitializePopulationTest.class,
- GAGridCalculateFitnessTest.class
-})
-
-public class GAGridTestSuite {
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/genetic/PasswordFitnessFunction.java b/modules/ml/src/test/java/org/apache/ignite/ml/genetic/PasswordFitnessFunction.java
deleted file mode 100644
index cb5aa00..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/genetic/PasswordFitnessFunction.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.genetic;
-
-import java.util.List;
-
-/**
- * PasswordFitnessFunction evaluates randomly generated passwords giving higher score to passwords with more special
- * characters.
- */
-public class PasswordFitnessFunction implements IFitnessFunction {
- /**
- * @param genes List of Genes within an individual Chromosome
- * @return Fitness score
- */
- @Override public double evaluate(List<Gene> genes) {
-
- double specialCCnt = 0;
- double lowerCaseCnt = 0;
- double upperCaseCnt = 0;
-
- double fitness;
- double specialCScore = 2;
-
- for (Gene gene : genes) {
- Character aCharacter = (Character)(gene.getVal());
- if (Character.isUpperCase(aCharacter))
- upperCaseCnt += 1;
-
- else if (Character.isLowerCase(aCharacter))
- lowerCaseCnt += 1;
- else
- specialCCnt += 1;
- }
-
- specialCCnt = specialCScore * specialCCnt;
- fitness = upperCaseCnt + lowerCaseCnt + specialCCnt;
- return fitness;
- }
-}
diff --git a/modules/ml/tensorflow-model-parser/pom.xml b/modules/ml/tensorflow-model-parser/pom.xml
deleted file mode 100644
index adc8364..0000000
--- a/modules/ml/tensorflow-model-parser/pom.xml
+++ /dev/null
@@ -1,88 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!--
- POM file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <properties>
- <tensorflow.version>1.12.0</tensorflow.version>
- </properties>
-
- <parent>
- <artifactId>ignite-parent</artifactId>
- <groupId>org.apache.ignite</groupId>
- <version>1</version>
- <relativePath>../../../parent</relativePath>
- </parent>
-
-
- <artifactId>ignite-ml-tensorflow-model-parser</artifactId>
- <version>2.9.0-SNAPSHOT</version>
- <url>http://ignite.apache.org</url>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-libs</id>
- <phase>package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <excludeTransitive>false</excludeTransitive>
- <excludeArtifactIds>
- fastutil,h2,ignite-shmem,annotations,arpack_combined_all,
- cache-api,commons-math3,commons-rng-client-api,commons-rng-core,
- commons-rng-simple,core,ignite-ml
- </excludeArtifactIds>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-ml</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.tensorflow</groupId>
- <artifactId>tensorflow</artifactId>
- <version>${tensorflow.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.tensorflow</groupId>
- <artifactId>proto</artifactId>
- <version>${tensorflow.version}</version>
- </dependency>
- </dependencies>
-</project>
diff --git a/modules/ml/tensorflow-model-parser/src/main/java/org/apache/ignite/ml/inference/parser/TensorFlowBaseModelParser.java b/modules/ml/tensorflow-model-parser/src/main/java/org/apache/ignite/ml/inference/parser/TensorFlowBaseModelParser.java
deleted file mode 100644
index b40486d..0000000
--- a/modules/ml/tensorflow-model-parser/src/main/java/org/apache/ignite/ml/inference/parser/TensorFlowBaseModelParser.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.inference.parser;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import org.apache.ignite.ml.inference.Model;
-import org.tensorflow.Session;
-import org.tensorflow.Tensor;
-
-/**
- * Base class for TensorFlow model parsers. Contains the logic that is common for models saved as "SavedModel" and as a
- * simple graph.
- *
- * @param <I> Type of model input.
- * @param <O> Type of model output.
- */
-public abstract class TensorFlowBaseModelParser<I, O> implements ModelParser<I, O, Model<I, O>> {
- /** */
- private static final long serialVersionUID = 5574259553625871456L;
-
- /** Map of input graph nodes (placeholders) and transformers that allow to transform input into tensor. */
- private final Map<String, InputTransformer<I>> inputs = new HashMap<>();
-
- /** List of output graph nodes. */
- private List<String> outputNames;
-
- /** Transformer that allows to transform tensors into output. */
- private OutputTransformer<O> outputTransformer;
-
- /** {@inheritDoc} */
- @Override public Model<I, O> parse(byte[] mdl) {
- return new TensorFlowInfModel(parseModel(mdl));
- }
-
- /**
- * Parses model specified in serialized form as byte array.
- *
- * @param mdl Inference model in serialized form as byte array.
- * @return TensorFlow session that encapsulates the TensorFlow graph parsed from serialized model.
- */
- public abstract Session parseModel(byte[] mdl);
-
- /**
- * Setter that allows to specify additional input graph node and correspondent transformer that allows to transform
- * input into tensor.
- *
- * @param name Name of the input graph node.
- * @param transformer Transformer that allows to transform input into tensor.
- * @return This instance.
- */
- public TensorFlowBaseModelParser<I, O> withInput(String name, InputTransformer<I> transformer) {
- if (inputs.containsKey(name))
- throw new IllegalArgumentException("Inputs already contains specified name [name=" + name + "]");
-
- inputs.put(name, transformer);
-
- return this;
- }
-
- /**
- * Setter that allows to specify output graph nodes and correspondent transformer that allow to transform tensors
- * into output.
- *
- * @param names List of output graph node names.
- * @param transformer Transformer that allow to transform tensors into output.
- * @return This instance.
- */
- public TensorFlowBaseModelParser<I, O> withOutput(List<String> names, OutputTransformer<O> transformer) {
- if (outputNames != null || outputTransformer != null)
- throw new IllegalArgumentException("Outputs already specified");
-
- outputNames = names;
- outputTransformer = transformer;
-
- return this;
- }
-
- /**
- * Input transformer that accepts input and transforms it into tensor.
- *
- * @param <I> Type of model input.
- */
- @FunctionalInterface
- public interface InputTransformer<I> extends Serializable {
- /**
- * Transforms input into tensor.
- *
- * @param input Input data.
- * @return Tensor (transformed input data).
- */
- public Tensor<?> transform(I input);
- }
-
- /**
- * Output transformer that accepts tensors and transforms them into output.
- *
- * @param <O> Type of model output.
- */
- @FunctionalInterface
- public interface OutputTransformer<O> extends Serializable {
- /**
- * Transforms tensors into output.
- *
- * @param output Tensors.
- * @return Output (transformed tensors).
- */
- public O transform(Map<String, Tensor<?>> output);
- }
-
- /**
- * TensorFlow inference model based on pre-loaded graph and created session.
- */
- private class TensorFlowInfModel implements Model<I, O> {
- /** TensorFlow session. */
- private final Session ses;
-
- /**
- * Constructs a new instance of TensorFlow inference model.
- *
- * @param ses TensorFlow session.
- */
- TensorFlowInfModel(Session ses) {
- this.ses = ses;
- }
-
- /** {@inheritDoc} */
- @Override public O predict(I input) {
- Session.Runner runner = ses.runner();
-
- runner = feedAll(runner, input);
- runner = fetchAll(runner);
-
- List<Tensor<?>> prediction = runner.run();
- Map<String, Tensor<?>> collectedPredictionTensors = indexTensors(prediction);
-
- return outputTransformer.transform(collectedPredictionTensors);
- }
-
- /**
- * Feeds input into graphs input nodes using input transformers (see {@link #inputs}).
- *
- * @param runner TensorFlow session runner.
- * @param input Input.
- * @return TensorFlow session runner.
- */
- private Session.Runner feedAll(Session.Runner runner, I input) {
- for (Map.Entry<String, InputTransformer<I>> e : inputs.entrySet()) {
- String opName = e.getKey();
- InputTransformer<I> transformer = e.getValue();
-
- runner = runner.feed(opName, transformer.transform(input));
- }
-
- return runner;
- }
-
- /**
- * Specifies graph output nodes to be fetched using {@link #outputNames}.
- *
- * @param runner TensorFlow session runner.
- * @return TensorFlow session runner.
- */
- private Session.Runner fetchAll(Session.Runner runner) {
- for (String e : outputNames)
- runner.fetch(e);
-
- return runner;
- }
-
- /**
- * Indexes tensors fetched from graph using {@link #outputNames}.
- *
- * @param tensors List of fetched tensors.
- * @return Map of tensor name as a key and tensor as a value.
- */
- private Map<String, Tensor<?>> indexTensors(List<Tensor<?>> tensors) {
- Map<String, Tensor<?>> collectedTensors = new HashMap<>();
-
- Iterator<String> outputNamesIter = outputNames.iterator();
- Iterator<Tensor<?>> tensorsIter = tensors.iterator();
-
- while (outputNamesIter.hasNext() && tensorsIter.hasNext())
- collectedTensors.put(outputNamesIter.next(), tensorsIter.next());
-
- // We expect that output names and output tensors have the same size.
- if (outputNamesIter.hasNext() || tensorsIter.hasNext())
- throw new IllegalStateException("Outputs are incorrect");
-
- return collectedTensors;
- }
-
- /** {@inheritDoc} */
- @Override public void close() {
- ses.close();
- }
- }
-}
diff --git a/modules/ml/tensorflow-model-parser/src/main/java/org/apache/ignite/ml/inference/parser/TensorFlowGraphModelParser.java b/modules/ml/tensorflow-model-parser/src/main/java/org/apache/ignite/ml/inference/parser/TensorFlowGraphModelParser.java
deleted file mode 100644
index dc1d99e..0000000
--- a/modules/ml/tensorflow-model-parser/src/main/java/org/apache/ignite/ml/inference/parser/TensorFlowGraphModelParser.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.inference.parser;
-
-import org.tensorflow.Graph;
-import org.tensorflow.Session;
-
-/**
- * Implementation of TensorFlow model parser that accepts serialized graph definition.
- *
- * @param <I> Type of model input.
- * @param <O> Type of model output.
- */
-public class TensorFlowGraphModelParser<I, O> extends TensorFlowBaseModelParser<I, O> {
- /** */
- private static final long serialVersionUID = -1872566748640565856L;
-
- /** {@inheritDoc} */
- @Override public Session parseModel(byte[] mdl) {
- Graph graph = new Graph();
- graph.importGraphDef(mdl);
-
- return new Session(graph);
- }
-}
diff --git a/modules/ml/tensorflow-model-parser/src/main/java/org/apache/ignite/ml/inference/parser/TensorFlowSavedModelModelParser.java b/modules/ml/tensorflow-model-parser/src/main/java/org/apache/ignite/ml/inference/parser/TensorFlowSavedModelModelParser.java
deleted file mode 100644
index 2c0b007..0000000
--- a/modules/ml/tensorflow-model-parser/src/main/java/org/apache/ignite/ml/inference/parser/TensorFlowSavedModelModelParser.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.inference.parser;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import org.apache.ignite.ml.inference.util.DirectorySerializer;
-import org.tensorflow.SavedModelBundle;
-import org.tensorflow.Session;
-
-/**
- * Implementation of TensorFlow model parser that accepts serialized directory with "SavedModel" as an input. The
- * directory is assumed to be serialized by {@link DirectorySerializer}.
- *
- * @param <I> Type of model input.
- * @param <O> Type of model output.
- */
-public class TensorFlowSavedModelModelParser<I, O> extends TensorFlowBaseModelParser<I, O> {
- /** */
- private static final long serialVersionUID = 5638083440240281879L;
-
- /** Prefix to be used to create temporary directory for TensorFlow model files. */
- private static final String TMP_DIR_PREFIX = "tensorflow_saved_model_";
-
- /** Model tags. */
- private final String[] tags;
-
- /**
- * Constructs a new instance of TensorFlow model parser.
- *
- * @param tags Model tags.
- */
- public TensorFlowSavedModelModelParser(String... tags) {
- this.tags = tags;
- }
-
- /** {@inheritDoc} */
- @Override public Session parseModel(byte[] mdl) {
- Path dir = null;
- try {
- dir = Files.createTempDirectory(TMP_DIR_PREFIX);
- DirectorySerializer.deserialize(dir.toAbsolutePath(), mdl);
- SavedModelBundle bundle = SavedModelBundle.load(dir.toString(), tags);
- return bundle.session();
- }
- catch (IOException | ClassNotFoundException e) {
- throw new RuntimeException(e);
- }
- finally {
- if (dir != null)
- DirectorySerializer.deleteDirectory(dir);
- }
- }
-}
diff --git a/modules/tensorflow/DEVNOTES.txt b/modules/tensorflow/DEVNOTES.txt
deleted file mode 100644
index 2fd9f95..0000000
--- a/modules/tensorflow/DEVNOTES.txt
+++ /dev/null
@@ -1 +0,0 @@
-TBD
\ No newline at end of file
diff --git a/modules/tensorflow/README.txt b/modules/tensorflow/README.txt
deleted file mode 100644
index 21ea88d..0000000
--- a/modules/tensorflow/README.txt
+++ /dev/null
@@ -1,80 +0,0 @@
-Apache Ignite TensorFlow Integration Module
-------------------------
-
-Apache Ignite TensorFlow Integration Module allowed using TensorFlow with Apache Ignite. In this scenario Apache Ignite
-will be a datasource for any TensorFlow model training.
-
-Import Apache Ignite TensorFlow Integration Module In Maven Project
--------------------------------------
-
-If you are using Maven to manage dependencies of your project, you can add TensorFlow module
-dependency like this (replace '${ignite.version}' with actual Ignite version you are
-interested in):
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
- http://maven.apache.org/xsd/maven-4.0.0.xsd">
- ...
- <dependencies>
- ...
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-tensorflow</artifactId>
- <version>${ignite.version}</version>
- </dependency>
- ...
- </dependencies>
- ...
-</project>
--------------------------------------
-
-TensorFlow integration module provides command line tool that allows to start, maintain and stop distributed deep
-learning utilizing Apache Ignite infrastructure and data. This tool provides several commands that are shown here:
-
-Usage: ignite-tf [-hV] [-c=<cfg>] [COMMAND]
-Apache Ignite and TensorFlow integration command line utility that allows to
-start, maintain and stop distributed deep learning utilizing Apache Ignite
-infrastructure and data.
- -c, --config=<cfg> Apache Ignite client configuration.
- -h, --help Show this help message and exit.
- -V, --version Print version information and exit.
-Commands:
- start Starts a new TensorFlow cluster and attaches to user script process.
- stop Stops a running TensorFlow cluster.
- attach Attaches to running TensorFlow cluster (user script process).
- ps Prints identifiers of all running TensorFlow clusters.
-
-To start TensorFlow cluster you need to specify upstream cache that will be used as data source for training, folder
-that contains code that actually performs training and command that should be called on this code to start training
-correctly. Command "start" have the following help output:
-
-Usage: ignite-tf start [-hV] [-c=<cfg>] CACHE_NAME JOB_DIR JOB_CMD [JOB_ARGS...]
-Starts a new TensorFlow cluster and attaches to user script process.
- CACHE_NAME Upstream cache name.
- JOB_DIR Job folder (or zip archive).
- JOB_CMD Job command.
- [JOB_ARGS...] Job arguments.
- -c, --config=<cfg> Apache Ignite client configuration.
- -h, --help Show this help message and exit.
- -V, --version Print version information and exit.
-
-To attach to running TensorFlow cluster or stop it you can use commands "attach" and "stop" correspondingly. These
-commands accepts cluster identifier as a parameter:
-
-Usage: ignite-tf attach [-hV] [-c=<cfg>] CLUSTER_ID
-Attaches to running TensorFlow cluster (user script process).
- CLUSTER_ID Cluster identifier.
- -c, --config=<cfg> Apache Ignite client configuration.
- -h, --help Show this help message and exit.
- -V, --version Print version information and exit.
-
-Usage: ignite-tf stop [-hV] [-c=<cfg>] CLUSTER_ID
-Stops a running TensorFlow cluster.
- CLUSTER_ID Cluster identifier.
- -c, --config=<cfg> Apache Ignite client configuration.
- -h, --help Show this help message and exit.
- -V, --version Print version information and exit.
-
-To find out what TensorFlow clusters are currently running on top of Apache Ignite you can use "ps" command that doesn't
-require arguments.
\ No newline at end of file
diff --git a/modules/tensorflow/licenses/apache-2.0.txt b/modules/tensorflow/licenses/apache-2.0.txt
deleted file mode 100644
index d645695..0000000
--- a/modules/tensorflow/licenses/apache-2.0.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/modules/tensorflow/pom.xml b/modules/tensorflow/pom.xml
deleted file mode 100644
index 9f45ae9..0000000
--- a/modules/tensorflow/pom.xml
+++ /dev/null
@@ -1,132 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!--
- POM file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>ignite-parent</artifactId>
- <groupId>org.apache.ignite</groupId>
- <version>1</version>
- <relativePath>../../parent/pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>ignite-tensorflow</artifactId>
- <version>2.9.0-SNAPSHOT</version>
- <url>http://ignite.apache.org</url>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-libs</id>
- <phase>package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <excludeTransitive>false</excludeTransitive>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-core</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-spring</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-slf4j</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-classic</artifactId>
- <version>1.2.3</version>
- </dependency>
-
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-core</artifactId>
- <version>1.2.3</version>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>${commons.io.version}</version>
- </dependency>
-
- <dependency>
- <groupId>info.picocli</groupId>
- <artifactId>picocli</artifactId>
- <version>3.3.0</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <profiles>
- <profile>
- <id>tensorflow</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-deploy-plugin</artifactId>
- <configuration>
- <skip>false</skip>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-</project>
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowCluster.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowCluster.java
deleted file mode 100644
index cf60e42..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowCluster.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster;
-
-import org.apache.ignite.tensorflow.cluster.spec.TensorFlowClusterSpec;
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-/**
- * TensorFlow Cluster metadata.
- */
-public class TensorFlowCluster implements Serializable {
- /** */
- private static final long serialVersionUID = -6636339457255751011L;
-
- /** TensorFlow cluster specification. */
- private final TensorFlowClusterSpec spec;
-
- /** Process identifiers. */
- private final Map<UUID, List<UUID>> processes;
-
- /**
- * Constructs a new instance of TensorFlow cluster.
- *
- * @param spec TensorFlow cluster specification.
- * @param processes Process identifiers.
- */
- public TensorFlowCluster(TensorFlowClusterSpec spec, Map<UUID, List<UUID>> processes) {
- assert spec != null : "TensorFlow cluster specification should not be null";
- assert processes != null : "Processes should not be null";
-
- this.spec = spec;
- this.processes = processes;
- }
-
- /** */
- public TensorFlowClusterSpec getSpec() {
- return spec;
- }
-
- /** */
- public Map<UUID, List<UUID>> getProcesses() {
- return processes;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGateway.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGateway.java
deleted file mode 100644
index 092dfcb..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGateway.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster;
-
-import java.util.HashSet;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.function.Consumer;
-import org.apache.ignite.lang.IgniteBiPredicate;
-import org.apache.ignite.tensorflow.util.SerializableConsumer;
-
-/**
- * TensorFlow cluster gateway that allows to subscribe on changes in cluster configuration.
- */
-public class TensorFlowClusterGateway implements IgniteBiPredicate<UUID, Optional<TensorFlowCluster>>, AutoCloseable {
- /** */
- private static final long serialVersionUID = -540323262800791340L;
-
- /** Callback that will be called on unsubscribe. */
- private final SerializableConsumer<TensorFlowClusterGateway> unsubscribeCb;
-
- /** Subscribers. */
- private final HashSet<Consumer<Optional<TensorFlowCluster>>> subscribers = new HashSet<>();
-
- /** Last value received from the upstream. */
- private Optional<TensorFlowCluster> last;
-
- /**
- * Constructs a new instance of TensorFlow cluster gateway.
- *
- * @param unsubscribeCb Callback that will be called on unsubscribe.
- */
- public TensorFlowClusterGateway(SerializableConsumer<TensorFlowClusterGateway> unsubscribeCb) {
- this.unsubscribeCb = unsubscribeCb;
- }
-
- /**
- * Subscribers the specified subscriber on the upstream events.
- *
- * @param subscriber Subscriber.
- */
- public synchronized void subscribe(Consumer<Optional<TensorFlowCluster>> subscriber) {
- subscribers.add(subscriber);
-
- if (last != null)
- subscriber.accept(last);
- }
-
- /**
- * Unsubscribe the specified subscriber.
- *
- * @param subscriber Subscriber.
- */
- public synchronized void unsubscribe(Consumer<Optional<TensorFlowCluster>> subscriber) {
- subscribers.remove(subscriber);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized boolean apply(UUID uuid, Optional<TensorFlowCluster> cluster) {
- for (Consumer<Optional<TensorFlowCluster>> subscriber : subscribers)
- subscriber.accept(cluster);
-
- last = cluster;
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public void close() {
- subscribers.clear();
- unsubscribeCb.accept(this);
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGatewayManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGatewayManager.java
deleted file mode 100644
index a315aaf..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGatewayManager.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster;
-
-import java.util.Optional;
-import java.util.UUID;
-import java.util.concurrent.CountDownLatch;
-import java.util.function.Consumer;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteLogger;
-
-/**
- * TensorFlow cluster manager that allows to start, maintain and stop TensorFlow cluster using
- * {@link TensorFlowClusterManager} and Ignite Service Grid.
- */
-public class TensorFlowClusterGatewayManager {
- /** Service name template. */
- private static final String SERVICE_NAME_TEMPLATE = "TF_SERVICE_%s";
-
- /** Service topic name template. */
- private static final String SERVICE_TOPIC_NAME_TEMPLATE = "TF_SERVICE_TOPIC_%s";
-
- /** Ignite instance. */
- private final Ignite ignite;
-
- /** Logger. */
- private final IgniteLogger log;
-
- /**
- * Constructs a new instance of TensorFlow cluster manager with maintenance.
- *
- * @param ignite Ignite instance.
- */
- public TensorFlowClusterGatewayManager(Ignite ignite) {
- assert ignite != null : "Ignite should not be null";
-
- this.ignite = ignite;
- this.log = ignite.log().getLogger(TensorFlowClusterGatewayManager.class);
- }
-
- /**
- * Subscribes on changes of the specified cluster.
- *
- * @param clusterId Cluster identifier.
- * @return TensorFlow cluster gateway that allows to subscribe on cluster changes.
- */
- public TensorFlowClusterGateway getCluster(UUID clusterId) {
- String topicName = String.format(SERVICE_TOPIC_NAME_TEMPLATE, clusterId);
-
- return createTensorFlowClusterGateway(topicName);
- }
-
- /**
- * Creates and starts a new TensorFlow cluster for the specified cache.
- *
- * @param clusterId Cluster identifier.
- * @param jobArchive Job archive.
- * @return TensorFlow cluster gateway that allows to subscribe on cluster changes.
- */
- public TensorFlowClusterGateway createCluster(UUID clusterId, TensorFlowJobArchive jobArchive) {
- String svcName = String.format(SERVICE_NAME_TEMPLATE, clusterId);
- String topicName = String.format(SERVICE_TOPIC_NAME_TEMPLATE, clusterId);
-
- TensorFlowClusterGateway gateway = createTensorFlowClusterGateway(topicName);
-
- ignite.services().deployClusterSingleton(
- svcName,
- new TensorFlowClusterMaintainer(clusterId, jobArchive, topicName)
- );
- log.info("Cluster maintainer deployed as a service [clusterId=" + clusterId + "]");
-
- return gateway;
- }
-
- /**
- * Listens to TensorFlow cluster user script.
- *
- * @param clusterId Cluster identifier.
- * @param out Output stream consumer.
- * @param err Error stream consumer.
- */
- public void listenToClusterUserScript(UUID clusterId, Consumer<String> out, Consumer<String> err) {
- TensorFlowClusterGateway gateway = getCluster(clusterId);
-
- ignite.message().localListen("us_out_" + clusterId, (node, msg) -> {
- out.accept(msg.toString());
- return true;
- });
-
- ignite.message().localListen("us_err_" + clusterId, (node, msg) -> {
- err.accept(msg.toString());
- return true;
- });
-
- CountDownLatch latch = new CountDownLatch(1);
-
- Consumer<Optional<TensorFlowCluster>> subscriber = cluster -> {
- if (!cluster.isPresent())
- latch.countDown();
- };
-
- gateway.subscribe(subscriber);
-
- try {
- latch.await();
- gateway.unsubscribe(subscriber);
- }
- catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Stops TensorFlow cluster.
- *
- * @param clusterId Cluster identifier.
- */
- public void stopClusterIfExists(UUID clusterId) {
- ignite.services().cancel(String.format(SERVICE_NAME_TEMPLATE, clusterId));
- log.info("Cluster maintained cancelled as a service [clusterId=" + clusterId + "]");
- }
-
- /**
- * Creates TensorFlow cluster gateway.
- *
- * @param topicName Topic name.
- * @return TensorFlow cluster gateway.
- */
- private TensorFlowClusterGateway createTensorFlowClusterGateway(String topicName) {
- TensorFlowClusterGateway gateway = new TensorFlowClusterGateway(subscriber -> {
- ignite.message().stopLocalListen(topicName, subscriber);
- log.info("Stop listen to cluster gateway [topicName=" + topicName + "]");
- });
-
- ignite.message().localListen(topicName, gateway);
- log.info("Start listen to cluster gateway [topicName=" + topicName + "]");
-
- return gateway;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterMaintainer.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterMaintainer.java
deleted file mode 100644
index f44a4f8..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterMaintainer.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.concurrent.locks.LockSupport;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.resources.LoggerResource;
-import org.apache.ignite.services.Service;
-import org.apache.ignite.services.ServiceContext;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessState;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
-
-/**
- * TensorFlow cluster service that maintains TensorFlow cluster.
- */
-public class TensorFlowClusterMaintainer implements Service {
- /** */
- private static final long serialVersionUID = -3220563310643566419L;
-
- /** Ignite instance. */
- @IgniteInstanceResource
- private transient Ignite ignite;
-
- /** Logger. */
- @LoggerResource
- private transient IgniteLogger log;
-
- /** TensorFlow cluster identifier. */
- private final UUID clusterId;
-
- /** Job archive. */
- private final TensorFlowJobArchive jobArchive;
-
- /** Topic name. */
- private final String topicName;
-
- /** TensorFlow cluster manager. */
- private transient TensorFlowClusterManager clusterMgr;
-
- /** Previous partition mapping. */
- private transient UUID[] prev;
-
- /**
- * Constructs a new instance of TensorFlow cluster service.
- *
- * @param clusterId Cluster identifier.
- * @param jobArchive Job archive.
- * @param topicName Topic name.
- */
- public TensorFlowClusterMaintainer(UUID clusterId, TensorFlowJobArchive jobArchive, String topicName) {
- assert clusterId != null : "Cluster identifier should not be null";
- assert jobArchive != null : "Job archive should not be null";
- assert topicName != null : "Topic name should not be null";
-
- this.clusterId = clusterId;
- this.jobArchive = jobArchive;
- this.topicName = topicName;
- }
-
- /** {@inheritDoc} */
- @Override public void cancel(ServiceContext ctx) {
- clusterMgr.stopClusterIfExists(clusterId);
- log.debug("Cluster maintainer canceled [clusterId=" + clusterId + "]");
- }
-
- /** {@inheritDoc} */
- @Override public void init(ServiceContext ctx) {
- clusterMgr = new TensorFlowClusterManager(ignite);
- log.debug("Cluster maintainer initialized [clusterId=" + clusterId + "]");
- }
-
- /** {@inheritDoc} */
- @Override public void execute(ServiceContext ctx) {
- while (!ctx.isCancelled() && !hasUserScriptCompletedSuccessfully()) {
- LockSupport.parkNanos(1_000_000);
-
- boolean restartRequired = hasAffinityChanged()
- || hasAnyWorkerFailed()
- || hasChiefFailed()
- || hasUserScriptFailed();
-
- if (restartRequired) {
- log.debug("Cluster will be restarted [clusterId=" + clusterId + "]");
-
- restartCluster();
- }
- }
-
- stopCluster(true);
-
- log.debug("Cluster maintainer completed [clusterId=" + clusterId + "]");
- }
-
- /**
- * Restarts TensorFlow cluster.
- */
- private void restartCluster() {
- stopCluster(false);
- startCluster();
- }
-
- /**
- * Stops TensorFlow cluster.
- *
- * @param terminate Terminate TensorFlow cluster and notify all listeners that cluster won't be started again.
- */
- private void stopCluster(boolean terminate) {
- clusterMgr.stopClusterIfExists(clusterId);
-
- if (terminate)
- ignite.message().send(topicName, Optional.empty());
- }
-
- /**
- * Starts TensorFlow cluster.
- */
- private void startCluster() {
- TensorFlowCluster cluster = clusterMgr.createCluster(
- clusterId,
- jobArchive,
- str -> ignite.message().sendOrdered("us_out_" + clusterId, str, 60 * 1000),
- str -> ignite.message().sendOrdered("us_err_" + clusterId, str, 60 * 1000)
- );
-
- ignite.message().send(topicName, Optional.of(cluster));
- }
-
- /**
- * Checks if affinity mapping has been changed.
- *
- * @return True if mapping has been changed, otherwise false.
- */
- private boolean hasAffinityChanged() {
- Affinity<?> affinity = ignite.affinity(jobArchive.getUpstreamCacheName());
-
- int parts = affinity.partitions();
-
- UUID[] ids = new UUID[parts];
-
- for (int part = 0; part < parts; part++) {
- ClusterNode node = affinity.mapPartitionToNode(part);
- UUID nodeId = node.id();
- ids[part] = nodeId;
- }
-
- if (prev == null || !Arrays.equals(ids, prev)) {
- prev = ids;
- return true;
- }
-
- return false;
- }
-
- /**
- * Checks is any worker has failed.
- *
- * @return True if any worker has failed, otherwise false.
- */
- private boolean hasAnyWorkerFailed() {
- TensorFlowCluster cluster = clusterMgr.getCluster(clusterId);
-
- Map<UUID, List<LongRunningProcessStatus>> statuses;
- try {
- statuses = clusterMgr.getSrvProcMgr().ping(cluster.getProcesses());
- }
- catch (Exception e) {
- log.error("Failed to check process statuses", e);
-
- return true;
- }
-
- for (UUID nodeId : statuses.keySet()) {
- for (LongRunningProcessStatus status : statuses.get(nodeId)) {
- if (status.getState().equals(LongRunningProcessState.DONE))
- return true;
- }
- }
-
- return false;
- }
-
- /**
- * Checks if chief has failed.
- *
- * @return True if chief has failed, otherwise false.
- */
- private boolean hasChiefFailed() {
- return clusterMgr.getChiefException(clusterId) != null;
- }
-
- /**
- * Checks if user script failed.
- *
- * @return True if user script has failed, otherwise false.
- */
- private boolean hasUserScriptFailed() {
- return clusterMgr.getUserScriptException(clusterId) != null;
- }
-
- /**
- * Checks if user script has completed successfully.
- *
- * @return True if user script has completed successfully, otherwise false.
- */
- private boolean hasUserScriptCompletedSuccessfully() {
- return clusterMgr.isUserScriptCompleted(clusterId)
- && clusterMgr.getUserScriptException(clusterId) == null;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterManager.java
deleted file mode 100644
index 12ed4ea..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterManager.java
+++ /dev/null
@@ -1,351 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.locks.Lock;
-import java.util.function.Consumer;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.ScanQuery;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.tensorflow.cluster.spec.TensorFlowClusterSpec;
-import org.apache.ignite.tensorflow.cluster.spec.TensorFlowServerAddressSpec;
-import org.apache.ignite.tensorflow.cluster.tfrunning.TensorFlowServer;
-import org.apache.ignite.tensorflow.cluster.tfrunning.TensorFlowServerManager;
-import org.apache.ignite.tensorflow.cluster.util.TensorFlowChiefRunner;
-import org.apache.ignite.tensorflow.cluster.util.TensorFlowClusterResolver;
-import org.apache.ignite.tensorflow.cluster.util.TensorFlowUserScriptRunner;
-import org.apache.ignite.tensorflow.core.util.CustomizableThreadFactory;
-
-/**
- * TensorFlow cluster manager that allows to start, maintain and stop TensorFlow cluster.
- */
-public class TensorFlowClusterManager {
- /** TensorFlow cluster metadata cache name. */
- private static final String TF_CLUSTER_METADATA_CACHE_NAME = "TF_CLUSTER_METADATA_CACHE";
-
- /** Ignite instance. */
- private final Ignite ignite;
-
- /** TensorFlow server manager. */
- private final TensorFlowServerManager srvProcMgr;
-
- /** TensorFlow cluster resolver. */
- private final TensorFlowClusterResolver clusterRslvr;
-
- /** TensorFlow cluster metadata cache. */
- private IgniteCache<UUID, TensorFlowCluster> cache;
-
- /** TensorFlow chief runners. */
- private ConcurrentMap<UUID, TensorFlowChiefRunner> chiefRunners;
-
- /** TensorFlow user script runners. */
- private ConcurrentMap<UUID, TensorFlowUserScriptRunner> userScriptRunners;
-
- /**
- * Constructs a new instance of TensorFlow cluster manager.
- *
- * @param ignite Ignite instance.
- */
- public TensorFlowClusterManager(Ignite ignite) {
- assert ignite != null : "Ignite instance should not be null";
-
- this.ignite = ignite;
- this.srvProcMgr = new TensorFlowServerManager(ignite);
- this.clusterRslvr = new TensorFlowClusterResolver(ignite, "TF", 10000, 1000);
- this.cache = getOrCreateCache();
- this.chiefRunners = new ConcurrentHashMap<>();
- this.userScriptRunners = new ConcurrentHashMap<>();
- }
-
- /**
- * Returns cluster by identifier.
- *
- * @param clusterId Cluster identifier.
- * @return TensorFlow cluster.
- */
- public TensorFlowCluster getCluster(UUID clusterId) {
- return cache.get(clusterId);
- }
-
- /**
- * Creates and starts a new TensorFlow cluster for the specified cache.
- *
- * @param clusterId Cluster identifier.
- * @param jobArchive Job archive.
- * @return TensorFlow cluster metadata.
- */
- public TensorFlowCluster createCluster(UUID clusterId, TensorFlowJobArchive jobArchive,
- Consumer<String> userScriptOut, Consumer<String> userScriptErr) {
- Lock clusterMgrCacheLock = cache.lock(clusterId);
- clusterMgrCacheLock.lock();
-
- try {
- TensorFlowCluster cluster = cache.get(clusterId);
-
- if (cluster != null)
- throw new IllegalStateException("Cluster is already created [clusterId=" + clusterId + "]");
-
- TensorFlowClusterSpec clusterSpec = clusterRslvr.resolveAndAcquirePorts(jobArchive.getUpstreamCacheName());
- cluster = startCluster(clusterId, clusterSpec, jobArchive, userScriptOut, userScriptErr);
- cache.put(clusterId, cluster);
-
- return cluster;
- }
- finally {
- clusterMgrCacheLock.unlock();
- }
- }
-
- /**
- * Stops TensorFlow cluster.
- *
- * @param clusterId TensorFlow cluster identifier.
- */
- public void stopClusterIfExists(UUID clusterId) {
- Lock clusterMgrCacheLock = cache.lock(clusterId);
- clusterMgrCacheLock.lock();
-
- try {
- TensorFlowCluster cluster = cache.get(clusterId);
-
- if (cluster != null) {
- stopChief(clusterId);
- stopUserScript(clusterId);
- srvProcMgr.stop(cluster.getProcesses(), true);
- clusterRslvr.releasePorts(cluster.getSpec());
- cache.remove(clusterId);
- }
- }
- finally {
- clusterMgrCacheLock.unlock();
- }
- }
-
- /** Destroys TensorFlow cluster manager and related caches. */
- public void destroy() {
- clusterRslvr.destroy();
- ignite.destroyCache(TF_CLUSTER_METADATA_CACHE_NAME);
- }
-
- /**
- * Starts TensorFlow cluster using the specified specification and returns metadata of the started cluster.
- *
- * @param clusterId Cluster identifier.
- * @param spec TensorFlow cluster specification.
- * @return TensorFlow cluster metadata.
- */
- private TensorFlowCluster startCluster(UUID clusterId, TensorFlowClusterSpec spec, TensorFlowJobArchive jobArchive,
- Consumer<String> userScriptOut, Consumer<String> userScriptErr) {
- Map<String, List<TensorFlowServerAddressSpec>> jobs = spec.getJobs();
-
- Map<UUID, List<UUID>> workerProcesses = startWorkers(spec, jobs.get(TensorFlowClusterResolver.WORKER_JOB_NAME));
-
- startChief(clusterId, spec);
- startUserScript(clusterId, jobArchive, spec, userScriptOut, userScriptErr);
-
- return new TensorFlowCluster(spec, workerProcesses);
- }
-
- /**
- * Starts TensorFlow worker processes using the specified specification and returns identifiers of the started
- * processes.
- *
- * @param spec TensorFlow cluster specification.
- * @param tasks Worker tasks.
- * @return Identifiers of the started processes.
- */
- private Map<UUID, List<UUID>> startWorkers(TensorFlowClusterSpec spec, List<TensorFlowServerAddressSpec> tasks) {
- List<TensorFlowServer> srvs = new ArrayList<>();
-
- if (tasks != null) {
- for (int i = 0; i < tasks.size(); i++) {
- TensorFlowServer srvSpec = new TensorFlowServer(spec, TensorFlowClusterResolver.WORKER_JOB_NAME, i);
- srvs.add(srvSpec);
- }
- }
-
- return srvProcMgr.start(srvs);
- }
-
- /**
- * Starts chief process using the specified cluster specification.
- *
- * @param clusterId Cluster identifier.
- * @param spec TensorFlow cluster specification.
- */
- private void startChief(UUID clusterId, TensorFlowClusterSpec spec) {
- TensorFlowChiefRunner chiefRunner = new TensorFlowChiefRunner(
- ignite,
- Executors.newSingleThreadExecutor(
- new CustomizableThreadFactory("tf-ch", true)
- ),
- spec,
- System.out::println,
- System.err::println
- );
-
- chiefRunner.start();
-
- chiefRunners.put(clusterId, chiefRunner);
- }
-
- /**
- * Stops chief process.
- *
- * @param clusterId Cluster identifier.
- */
- private void stopChief(UUID clusterId) {
- TensorFlowChiefRunner runner = chiefRunners.remove(clusterId);
-
- if (runner != null)
- runner.stop();
- }
-
- /**
- * Checks if chief completed and returns result.
- *
- * @param clusterId Cluster identifier.
- * @return {@code true} if chief completed, otherwise {@code false}.
- */
- public boolean isChiefCompleted(UUID clusterId) {
- TensorFlowChiefRunner runner = chiefRunners.get(clusterId);
-
- return runner != null && runner.isCompleted();
- }
-
- /**
- * Returns an exception that happened during execution or {@code null} if there is no exception.
- *
- * @param clusterId Cluster identifier.
- * @return Exception that happened during execution or {@code null} if there is no exception.
- */
- public Exception getChiefException(UUID clusterId) {
- TensorFlowChiefRunner runner = chiefRunners.get(clusterId);
-
- return runner != null ? runner.getException() : null;
- }
-
- /**
- * Starts user script processes using the specified job archive.
- *
- * @param clusterId Cluster identifier.
- * @param jobArchive Job archive.
- * @param clusterSpec Cluster specification.
- */
- private void startUserScript(UUID clusterId, TensorFlowJobArchive jobArchive, TensorFlowClusterSpec clusterSpec,
- Consumer<String> out, Consumer<String> err) {
- TensorFlowUserScriptRunner userScriptRunner = new TensorFlowUserScriptRunner(
- ignite,
- Executors.newSingleThreadExecutor(
- new CustomizableThreadFactory("tf-us", true)
- ),
- jobArchive,
- clusterSpec,
- out,
- err
- );
-
- userScriptRunner.start();
-
- userScriptRunners.put(clusterId, userScriptRunner);
- }
-
- /**
- * Stops user script process.
- *
- * @param clusterId Cluster identifier.
- */
- private void stopUserScript(UUID clusterId) {
- TensorFlowUserScriptRunner runner = userScriptRunners.remove(clusterId);
-
- if (runner != null)
- runner.stop();
- }
-
- /**
- * Checks if user script completed and returns result.
- *
- * @param clusterId Cluster identifier.
- * @return {@code true} if user script completed, otherwise {@code false}.
- */
- public boolean isUserScriptCompleted(UUID clusterId) {
- TensorFlowUserScriptRunner runner = userScriptRunners.get(clusterId);
-
- return runner != null && runner.isCompleted();
- }
-
- /**
- * Returns an exception that happened during execution or {@code null} if there is no exception.
- *
- * @param clusterId Cluster identifier.
- * @return Exception that happened during execution or {@code null} if there is no exception.
- */
- public Exception getUserScriptException(UUID clusterId) {
- TensorFlowUserScriptRunner runner = userScriptRunners.get(clusterId);
-
- return runner != null ? runner.getException() : null;
- }
-
- /**
- * Returns list of maintained TensorFlow clusters.
- *
- * @return List of maintained TensorFlow clusters.
- */
- public Map<UUID, TensorFlowCluster> getAllClusters() {
- Map<UUID, TensorFlowCluster> res = new HashMap<>();
-
- ScanQuery<UUID, TensorFlowCluster> qry = new ScanQuery<>();
- QueryCursor<Cache.Entry<UUID, TensorFlowCluster>> cursor = cache.query(qry);
- for (Cache.Entry<UUID, TensorFlowCluster> e : cursor)
- res.put(e.getKey(), e.getValue());
-
- return res;
- }
-
- /** */
- public TensorFlowServerManager getSrvProcMgr() {
- return srvProcMgr;
- }
-
- /**
- * Returns existing cluster manager cache or creates a new one.
- *
- * @return Cluster manager cache.
- */
- private IgniteCache<UUID, TensorFlowCluster> getOrCreateCache() {
- CacheConfiguration<UUID, TensorFlowCluster> cacheConfiguration = new CacheConfiguration<>();
- cacheConfiguration.setName(TF_CLUSTER_METADATA_CACHE_NAME);
- cacheConfiguration.setCacheMode(CacheMode.REPLICATED);
- cacheConfiguration.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
-
- return ignite.getOrCreateCache(cacheConfiguration);
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowJobArchive.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowJobArchive.java
deleted file mode 100644
index 953cf76..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowJobArchive.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster;
-
-import java.io.Serializable;
-
-/**
- * TensorFlow job archive that keeps archived working directory and command to be executed.
- */
-public class TensorFlowJobArchive implements Serializable {
- /** */
- private static final long serialVersionUID = -5977231383594482459L;
-
- /** Upstream cache name. */
- private final String upstreamCacheName;
-
- /** Archived working directory. */
- private final byte[] data;
-
- /** Command to be executed with arguments. */
- private final String[] commands;
-
- /**
- * Constructs a new instance of TensorFlow job archive.
- *
- * @param upstreamCacheName Upstream cache name.
- * @param data Archived working directory.
- * @param commands Command to be executed with arguments.
- */
- public TensorFlowJobArchive(String upstreamCacheName, byte[] data, String[] commands) {
- this.upstreamCacheName = upstreamCacheName;
- this.data = data;
- this.commands = commands;
- }
-
- /** */
- public String getUpstreamCacheName() {
- return upstreamCacheName;
- }
-
- /** */
- public byte[] getData() {
- return data;
- }
-
- /** */
- public String[] getCommands() {
- return commands;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/package-info.java
deleted file mode 100644
index 858aa90..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/package-info.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * TensorFlow integration API that allows to start and maintain TensorFlow cluster using infrastructure tools from
- * package {@link org.apache.ignite.tensorflow.core}. The most important components are:
- * <ul>
- * <li>{@link org.apache.ignite.tensorflow.cluster.TensorFlowClusterManager} that allows to start and stop
- * TensorFlow cluster on top of Apache Ignite, but doesn't monitor it and doesn't maintain so that in case of
- * failure the cluster won't be restarted.</li>
- * <li>{@link org.apache.ignite.tensorflow.cluster.TensorFlowClusterGatewayManager} that allows to start, maintain
- * and stop TensorFlow cluster on top of Apache Ignite so that in case of failure the cluster will be restarted and
- * recovered.</li>
- * <li>{@link org.apache.ignite.tensorflow.cluster.TensorFlowClusterGateway} that allows to subscribe on cluster
- * configuration changes that might be done as result of rebalancing or node failures.</li>
- * </ul>
- */
-
-package org.apache.ignite.tensorflow.cluster;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowClusterSpec.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowClusterSpec.java
deleted file mode 100644
index ea813ea..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowClusterSpec.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster.spec;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.ignite.Ignite;
-
-/**
- * TensorFlow cluster specification.
- */
-public class TensorFlowClusterSpec implements Serializable {
- /** */
- private static final long serialVersionUID = 1428581667528448091L;
-
- /** TensorFlow cluster jobs. */
- private final Map<String, List<TensorFlowServerAddressSpec>> jobs = new HashMap<>();
-
- /**
- * Adds new task to the cluster specification.
- *
- * @param jobName Job name.
- * @param nodeId Node identifier.
- * @param port Port number.
- * @return This instance of TensorFlow cluster specification.
- */
- public TensorFlowClusterSpec addTask(String jobName, UUID nodeId, int port) {
- jobs.putIfAbsent(jobName, new ArrayList<>());
-
- List<TensorFlowServerAddressSpec> tasks = jobs.get(jobName);
-
- tasks.add(new TensorFlowServerAddressSpec(nodeId, port));
-
- return this;
- }
-
- /**
- * Formats cluster specification so that TensorFlow accepts it.
- *
- * @param ignite Ignite instance.
- * @return Formatted cluster specification.
- */
- public String format(Ignite ignite) {
- StringBuilder builder = new StringBuilder();
-
- builder.append("{\n");
-
- for (Map.Entry<String, List<TensorFlowServerAddressSpec>> entry : jobs.entrySet()) {
- builder
- .append("\t\"")
- .append(entry.getKey())
- .append("\" : [ ");
-
- for (TensorFlowServerAddressSpec address : entry.getValue()) {
- builder
- .append("\n\t\t\"")
- .append(address.format(ignite))
- .append("\", ");
- }
-
- if (!entry.getValue().isEmpty())
- builder.delete(builder.length() - 2, builder.length());
-
- builder.append("\n\t],\n");
- }
-
- if (!jobs.isEmpty())
- builder.delete(builder.length() - 2, builder.length() - 1);
-
- builder.append('}');
-
- return builder.toString();
- }
-
- /** */
- public Map<String, List<TensorFlowServerAddressSpec>> getJobs() {
- return jobs;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowServerAddressSpec.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowServerAddressSpec.java
deleted file mode 100644
index 9421874..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowServerAddressSpec.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster.spec;
-
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.UUID;
-import org.apache.ignite.Ignite;
-
-/**
- * TensorFlow server address specification.
- */
-public class TensorFlowServerAddressSpec implements Serializable {
- /** */
- private static final long serialVersionUID = 7883701602323727681L;
-
- /** Node identifier. */
- private final UUID nodeId;
-
- /** Port. */
- private final int port;
-
- /**
- * Constructs a new instance of TensorFlow server address specification.
- *
- * @param nodeId Node identifier.
- * @param port Port.
- */
- TensorFlowServerAddressSpec(UUID nodeId, int port) {
- assert nodeId != null : "Node identifier should not be null";
- assert port >= 0 && port <= 0xFFFF : "Port should be between 0 and 65535";
-
- this.nodeId = nodeId;
- this.port = port;
- }
-
- /**
- * Formats Server Address specification so that TensorFlow accepts it.
- *
- * @param ignite Ignite instance.
- * @return Formatted server address specification.
- */
- public String format(Ignite ignite) {
- Collection<String> names = ignite.cluster().forNodeId(nodeId).hostNames();
- return names.iterator().next() + ":" + port;
- }
-
- /** */
- public UUID getNodeId() {
- return nodeId;
- }
-
- /** */
- public int getPort() {
- return port;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/package-info.java
deleted file mode 100644
index f744ff9..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains specifications that describes TensorFlow cluster configuration.
- */
-
-package org.apache.ignite.tensorflow.cluster.spec;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServer.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServer.java
deleted file mode 100644
index ca67405..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServer.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster.tfrunning;
-
-import org.apache.ignite.tensorflow.cluster.spec.TensorFlowClusterSpec;
-import java.io.Serializable;
-
-/**
- * TensorFlow server specification.
- */
-public class TensorFlowServer implements Serializable {
- /** */
- private static final long serialVersionUID = 165988934166176805L;
-
- /** TensorFlow cluster specification. */
- private final TensorFlowClusterSpec clusterSpec;
-
- /** Job name. */
- private final String jobName;
-
- /** Task index. */
- private final Integer taskIdx;
-
- /** Protocol. */
- private final String proto;
-
- /**
- * Constructs a new instance of TensorFlow server specification.
- *
- * @param clusterSpec TensorFlow cluster specification.
- * @param jobName Job name.
- */
- public TensorFlowServer(TensorFlowClusterSpec clusterSpec, String jobName) {
- this(clusterSpec, jobName, null);
- }
-
- /**
- * Constructs a new instance of TensorFlow server specification.
- *
- * @param clusterSpec TensorFlow cluster specification.
- * @param jobName Job name.
- * @param taskIdx Task index.
- */
- public TensorFlowServer(TensorFlowClusterSpec clusterSpec, String jobName, Integer taskIdx) {
- this(clusterSpec, jobName, taskIdx, null);
- }
-
- /**
- * Constructs a new instance of TensorFlow server specification.
- *
- * @param clusterSpec TensorFlow cluster specification.
- * @param jobName Job name.
- * @param taskIdx Task index.
- * @param proto Task index.
- */
- public TensorFlowServer(TensorFlowClusterSpec clusterSpec, String jobName, Integer taskIdx, String proto) {
- assert clusterSpec != null : "TensorFlow cluster specification should not be null";
- assert jobName != null : "Job name should not be null";
-
- this.clusterSpec = clusterSpec;
- this.jobName = jobName;
- this.taskIdx = taskIdx;
- this.proto = proto;
- }
-
- /** */
- public TensorFlowClusterSpec getClusterSpec() {
- return clusterSpec;
- }
-
- /** */
- public String getJobName() {
- return jobName;
- }
-
- /** */
- public Integer getTaskIdx() {
- return taskIdx;
- }
-
- /** */
- public String getProto() {
- return proto;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerManager.java
deleted file mode 100644
index 589d1d7..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerManager.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster.tfrunning;
-
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.tensorflow.cluster.spec.TensorFlowClusterSpec;
-import org.apache.ignite.tensorflow.cluster.spec.TensorFlowServerAddressSpec;
-import org.apache.ignite.tensorflow.cluster.util.TensorFlowProcessBuilderSupplier;
-import org.apache.ignite.tensorflow.core.ProcessManager;
-import org.apache.ignite.tensorflow.core.ProcessManagerWrapper;
-import org.apache.ignite.tensorflow.core.nativerunning.NativeProcess;
-import org.apache.ignite.tensorflow.core.nativerunning.NativeProcessManager;
-
-/**
- * TensorFlow server manager that allows to start, stop and make other actions with TensorFlow servers.
- */
-public class TensorFlowServerManager extends ProcessManagerWrapper<NativeProcess, TensorFlowServer> {
- /** TensorFlow server script formatter. */
- private static final TensorFlowServerScriptFormatter scriptFormatter = new TensorFlowServerScriptFormatter();
-
- /**
- * Constructs a new instance of TensorFlow server manager.
- *
- * @param ignite Ignite instance.
- */
- public TensorFlowServerManager(Ignite ignite) {
- this(new NativeProcessManager(ignite));
- }
-
- /**
- * Constructs a new instance of TensorFlow server manager.
- *
- * @param delegate Delegate.
- */
- public TensorFlowServerManager(ProcessManager<NativeProcess> delegate) {
- super(delegate);
- }
-
- /** {@inheritDoc} */
- @Override protected NativeProcess transformSpecification(TensorFlowServer spec) {
- return new NativeProcess(
- new TensorFlowProcessBuilderSupplier(
- true,
- true,
- "job:" + spec.getJobName(),
- "task:" + spec.getTaskIdx()
- ),
- scriptFormatter.format(spec, true, Ignition.ignite()),
- getNode(spec)
- );
- }
-
- /**
- * Extracts the cluster node server should be running on.
- *
- * @param spec TensorFlow server specification.
- * @return Node identifier.
- */
- private UUID getNode(TensorFlowServer spec) {
- TensorFlowClusterSpec clusterSpec = spec.getClusterSpec();
- Map<String, List<TensorFlowServerAddressSpec>> jobs = clusterSpec.getJobs();
- List<TensorFlowServerAddressSpec> tasks = jobs.get(spec.getJobName());
- TensorFlowServerAddressSpec addr = tasks.get(spec.getTaskIdx());
-
- return addr.getNodeId();
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerScriptFormatter.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerScriptFormatter.java
deleted file mode 100644
index fc90ef0..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerScriptFormatter.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster.tfrunning;
-
-import org.apache.ignite.Ignite;
-
-/**
- * Utils class that helps to format Python script that starts TensorFlow server.
- */
-public class TensorFlowServerScriptFormatter {
- /**
- * Formats TensorFlow server specification so that it's available to be passed into а python script.
- *
- * @param srv Server specification.
- * @param join Joins server by default or not.
- * @param ignite Ignite instance.
- * @return Formatted TensorFlow server script.
- */
- public String format(TensorFlowServer srv, boolean join, Ignite ignite) {
- StringBuilder builder = new StringBuilder();
-
- builder.append("from __future__ import absolute_import").append("\n");
- builder.append("from __future__ import division").append("\n");
- builder.append("from __future__ import print_function").append("\n");
-
- builder.append("from threading import Thread").append("\n");
- builder.append("from time import sleep").append("\n");
- builder.append("import os, signal").append("\n");
-
- builder.append("\n");
-
- builder.append("def check_pid(pid):").append("\n");
- builder.append(" try:").append("\n");
- builder.append(" os.kill(pid, 0)").append("\n");
- builder.append(" except OSError:").append("\n");
- builder.append(" return False").append("\n");
- builder.append(" else:").append("\n");
- builder.append(" return True").append("\n");
-
- builder.append("\n");
-
- builder.append("def threaded_function(pid):").append("\n");
- builder.append(" while check_pid(pid):").append("\n");
- builder.append(" sleep(1)").append("\n");
- builder.append(" os.kill(os.getpid(), signal.SIGUSR1)").append("\n");
-
- builder.append("\n");
-
- builder.append("Thread(target = threaded_function, args = (int(os.environ['PPID']), )).start()")
- .append("\n");
- builder.append("\n");
-
- builder.append("import tensorflow as tf").append('\n');
- builder.append("to_import_contrib_ops = tf.contrib.resampler").append("\n");
- builder.append("from tensorflow.contrib.ignite import IgniteDataset").append("\n");
- builder.append("import tensorflow.contrib.igfs.python.ops.igfs_ops").append("\n");
-
- builder.append("print('job:%s task:%d' % ('")
- .append(srv.getJobName())
- .append("', ")
- .append(srv.getTaskIdx())
- .append("))")
- .append("\n");
- builder.append("print('IGNITE_DATASET_HOST = ', os.environ.get('IGNITE_DATASET_HOST'))").append("\n");
- builder.append("print('IGNITE_DATASET_PORT = ', os.environ.get('IGNITE_DATASET_PORT'))").append("\n");
- builder.append("print('IGNITE_DATASET_LOCAL = ', os.environ.get('IGNITE_DATASET_LOCAL'))").append("\n");
-
- builder.append("os.environ['TF_CONFIG'] = '").append(formatTfConfigVar(srv, ignite)).append("'\n");
- builder.append("server = tf.contrib.distribute.run_standard_tensorflow_server()").append("\n");
-
- if (join)
- builder.append("server.join()").append('\n');
-
- return builder.toString();
- }
-
- /**
- * Formats "TF_CONFIG" variable to be passed into user script.
- *
- * @param srv Server description.
- * @param ignite Ignite instance.
- * @return Formatted "TF_CONFIG" variable to be passed into user script.
- */
- private String formatTfConfigVar(TensorFlowServer srv, Ignite ignite) {
- return "{\"cluster\" : " +
- srv.getClusterSpec().format(ignite).replace('\n', ' ') +
- ", " +
- "\"task\": {\"type\" : \"" +
- srv.getJobName() +
- "\", \"index\": " +
- srv.getTaskIdx() +
- "}}";
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/package-info.java
deleted file mode 100644
index b226bb1..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Additional "running" layer on top of Python running layer of infrastructure. Allows to start TensorFlow server using
- * server description ({@link org.apache.ignite.tensorflow.cluster.tfrunning.TensorFlowServer}).
- */
-
-package org.apache.ignite.tensorflow.cluster.tfrunning;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/ClusterPortManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/ClusterPortManager.java
deleted file mode 100644
index 681e07f..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/ClusterPortManager.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster.util;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cluster.ClusterGroup;
-import org.apache.ignite.cluster.ClusterGroupEmptyException;
-import org.apache.ignite.configuration.CacheConfiguration;
-
-import java.io.Serializable;
-import java.net.NetworkInterface;
-import java.util.*;
-import java.util.concurrent.locks.Lock;
-
-/**
- * Cluster port manager that allows to reliably {@link #acquirePort(UUID)} and {@link #releasePort(UUID, int)} on the
- * cluster nodes.
- */
-public class ClusterPortManager {
- /** Ignite instance. */
- private final Ignite ignite;
-
- /** Ignite logger. */
- private final IgniteLogger log;
-
- /** Port manager cache name. */
- private final String portMgrCacheName;
-
- /** Port range from point. */
- private final int from;
-
- /** Port range size. */
- private final int cnt;
-
- /** Port manager cache */
- private final IgniteCache<HostIdentifier, BitSet> cache;
-
- /**
- * Constructs a new instance of cluster port manager.
- *
- * @param poolName Port pool name.
- * @param from Port range from point.
- * @param cnt Port range size.
- */
- public ClusterPortManager(Ignite ignite, String poolName, int from, int cnt) {
- assert ignite != null : "Ignite instance should not be null";
- assert poolName != null : "Pool name should not be null";
- assert cnt >= 0 : "Count should not be negative";
- assert from >= 0 && cnt + from <= 0xFFFF : "Port range should be between 0 and 65535";
-
- this.ignite = ignite;
- this.log = ignite.log().getLogger(ClusterPortManager.class);
-
- this.portMgrCacheName = String.format("PORT_MANAGER_%s_CACHE", poolName);
- this.from = from;
- this.cnt = cnt;
- this.cache = getOrCreateCache();
- }
-
- /**
- * Acquires free port on the specified node.
- *
- * @param nodeId Node identifier.
- * @return Port to be acquired.
- */
- public int acquirePort(UUID nodeId) {
- HostIdentifier hostId = getHostIdentifier(nodeId);
-
- if (hostId == null)
- throw new IllegalStateException("Can't find node [nodeId=" + nodeId + "]");
-
- Lock lock = cache.lock(hostId);
- lock.lock();
-
- try {
- BitSet ports = cache.get(hostId);
-
- if (ports == null)
- ports = new BitSet(cnt);
-
- int free = ports.nextClearBit(0);
-
- if (free >= cnt)
- throw new IllegalStateException("No free ports in range [from=" + from + ", cnt=" + cnt + "]");
-
- ports.set(free);
- log.debug("Port acquired [nodeId=" + nodeId + ", port=" + (from + free) + "]");
-
- cache.put(hostId, ports);
-
- return from + free;
- }
- finally {
- lock.unlock();
- }
- }
-
- /**
- * Releases acquired port on the specified node.
- *
- * @param nodeId Node identifier.
- * @param port Acquired port to be free.
- */
- public void releasePort(UUID nodeId, int port) {
- assert port - from >= 0 && port - from < cnt : "Port not in the range";
-
- HostIdentifier hostId = getHostIdentifier(nodeId);
-
- if (hostId == null)
- return;
-
- Lock lock = cache.lock(hostId);
- lock.lock();
-
- try {
- BitSet ports = cache.get(hostId);
-
- if (ports != null) {
- ports.clear(port - from);
- log.debug("Port released [nodeId=" + nodeId + ", port=" + port + "]");
-
- if (ports.isEmpty())
- cache.remove(hostId);
- else
- cache.put(hostId, ports);
- }
- }
- finally {
- lock.unlock();
- }
- }
-
- /** Destroys port manager and related caches. */
- public void destroy() {
- ignite.destroyCache(portMgrCacheName);
- }
-
- /**
- * Returns existed port pool cache or creates a new one.
- *
- * @return Port pool cache.
- */
- private IgniteCache<HostIdentifier, BitSet> getOrCreateCache() {
- CacheConfiguration<HostIdentifier, BitSet> cacheConfiguration = new CacheConfiguration<>();
- cacheConfiguration.setName(portMgrCacheName);
- cacheConfiguration.setCacheMode(CacheMode.REPLICATED);
- cacheConfiguration.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
-
- return ignite.getOrCreateCache(cacheConfiguration);
- }
-
- /**
- * Returns host identifier by node identifier.
- *
- * @param nodeId Node identifier.
- * @return Host identifier.
- */
- private HostIdentifier getHostIdentifier(UUID nodeId) {
- try {
- ClusterGroup grp = ignite.cluster().forNodeId(nodeId);
-
- return ignite.compute(grp).call(() -> {
- Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces();
-
- List<byte[]> macAddrs = new ArrayList<>();
-
- while (interfaces.hasMoreElements()) {
- NetworkInterface netItf = interfaces.nextElement();
- byte[] macAddr = netItf.getHardwareAddress();
- macAddrs.add(macAddr);
- }
-
- return new HostIdentifier(macAddrs.toArray(new byte[macAddrs.size()][]));
- });
- }
- catch (ClusterGroupEmptyException e) {
- return null;
- }
- }
-
- /**
- * Host identifier based on arrays of mac addresses of the host machine.
- */
- private static class HostIdentifier implements Serializable {
- /** */
- private static final long serialVersionUID = -7060231325908935162L;
-
- /** Mac addresses. */
- private final byte[][] macAddrs;
-
- /**
- * Constructs a new instance of host identifier.
- *
- * @param macAddrs Mac addresses.
- */
- public HostIdentifier(byte[][] macAddrs) {
- this.macAddrs = macAddrs;
- }
-
- /** */
- public byte[][] getMacAddrs() {
- return macAddrs;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- HostIdentifier that = (HostIdentifier)o;
- if (macAddrs.length != that.macAddrs.length)
- return false;
-
- for (int i = 0; i < macAddrs.length; i++)
- if (!Arrays.equals(macAddrs[i], that.macAddrs[i]))
- return false;
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return Arrays.hashCode(macAddrs);
- }
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowChiefRunner.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowChiefRunner.java
deleted file mode 100644
index d8640fa..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowChiefRunner.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster.util;
-
-import java.util.concurrent.ExecutorService;
-import java.util.function.Consumer;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.tensorflow.cluster.spec.TensorFlowClusterSpec;
-import org.apache.ignite.tensorflow.cluster.tfrunning.TensorFlowServer;
-import org.apache.ignite.tensorflow.cluster.tfrunning.TensorFlowServerScriptFormatter;
-import org.apache.ignite.tensorflow.core.util.AsyncNativeProcessRunner;
-import org.apache.ignite.tensorflow.core.util.NativeProcessRunner;
-import org.apache.ignite.tensorflow.core.util.PythonProcessBuilderSupplier;
-
-/**
- * Utils class that helps to start and stop chief process.
- */
-public class TensorFlowChiefRunner extends AsyncNativeProcessRunner {
- /** Ignite instance. */
- private final Ignite ignite;
-
- /** TensorFlow cluster specification. */
- private final TensorFlowClusterSpec spec;
-
- /** Output stream data consumer. */
- private final Consumer<String> out;
-
- /** Error stream data consumer. */
- private final Consumer<String> err;
-
- /**
- * Constructs a new instance of TensorFlow chief runner.
- *
- * @param ignite Ignite instance.
- * @param executor Executor to be used in {@link AsyncNativeProcessRunner}.
- * @param spec TensorFlow cluster specification.
- * @param out Output stream data consumer.
- * @param err Error stream data consumer.
- */
- public TensorFlowChiefRunner(Ignite ignite, ExecutorService executor, TensorFlowClusterSpec spec,
- Consumer<String> out, Consumer<String> err) {
- super(ignite, executor);
- this.ignite = ignite;
- this.spec = spec;
- this.out = out;
- this.err = err;
- }
-
- /** {@inheritDoc} */
- @Override public NativeProcessRunner doBefore() {
- TensorFlowServer srv = new TensorFlowServer(spec, TensorFlowClusterResolver.CHIEF_JOB_NAME, 0);
-
- return new NativeProcessRunner(
- new PythonProcessBuilderSupplier(
- true,
- "job:" + srv.getJobName(),
- "task:" + srv.getTaskIdx()
- ).get(),
- new TensorFlowServerScriptFormatter().format(srv, true, ignite),
- out,
- err
- );
- }
-
- /** {@inheritDoc} */
- @Override public void doAfter() {
- // Do nothing.
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowClusterResolver.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowClusterResolver.java
deleted file mode 100644
index 87aafaf..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowClusterResolver.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster.util;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.tensorflow.cluster.spec.TensorFlowClusterSpec;
-import org.apache.ignite.tensorflow.cluster.spec.TensorFlowServerAddressSpec;
-
-/**
- * TensorFlow cluster resolver based on Ignite Cache affinity.
- */
-public class TensorFlowClusterResolver {
- /** TensorFlow worker job name. */
- public static final String WORKER_JOB_NAME = "worker";
-
- /** TensorFlow chief job name. */
- public static final String CHIEF_JOB_NAME = "chief";
-
- /** Ignite instance. */
- private final Ignite ignite;
-
- /** Cluster port manager. */
- private final ClusterPortManager portMgr;
-
- /**
- * Constructs a new instance of TensorFlow cluster resolver.
- *
- * @param ignite Ignite instance.
- */
- public TensorFlowClusterResolver(Ignite ignite, String portPoolName, int portFrom, int portCnt) {
- assert ignite != null : "Ignite instance should not be null";
- assert portPoolName != null : "Port pool name should not be null";
- assert portFrom >= 0 : "Port count should not be negative";
- assert portCnt >= 0 && portCnt + portFrom <= 0xFFFF : "Port range should be between 0 and 65535";
-
- this.ignite = ignite;
- this.portMgr = new ClusterPortManager(ignite, portPoolName, portFrom, portCnt);
- }
-
- /**
- * Resolves TensorFlow cluster and acquires required ports.
- *
- * @param upstreamCacheName Upstream cache name.
- * @return TensorFlow cluster specification.
- */
- public TensorFlowClusterSpec resolveAndAcquirePorts(String upstreamCacheName) {
- TensorFlowClusterSpec spec = new TensorFlowClusterSpec();
-
- resolveAndAcquirePortsForWorkers(spec, upstreamCacheName);
- resolveAndAcquirePortsForChief(spec);
-
- return spec;
- }
-
- /**
- * Releases ports acquired for the given cluster specification.
- *
- * @param spec TensorFlow cluster specification.
- */
- public void releasePorts(TensorFlowClusterSpec spec) {
- for (String jobName : spec.getJobs().keySet())
- for (TensorFlowServerAddressSpec address : spec.getJobs().get(jobName))
- portMgr.releasePort(address.getNodeId(), address.getPort());
- }
-
- /** Destroys TensorFlow cluster resolver. */
- public void destroy() {
- portMgr.destroy();
- }
-
- /**
- * Resolves TensorFlow cluster worker jobs and acquires ports.
- *
- * @param spec TensorFlow cluster specification.
- * @param upstreamCacheName Upstream cache name.
- */
- private void resolveAndAcquirePortsForWorkers(TensorFlowClusterSpec spec, String upstreamCacheName) {
- Affinity<?> affinity = ignite.affinity(upstreamCacheName);
- int parts = affinity.partitions();
-
- Set<UUID> distinctNodeIds = new HashSet<>();
- for (int part = 0; part < parts; part++) {
- ClusterNode node = affinity.mapPartitionToNode(part);
- UUID nodeId = node.id();
- distinctNodeIds.add(nodeId);
- }
- List<UUID> nodeIds = new ArrayList<>(distinctNodeIds);
- Collections.sort(nodeIds);
-
- for (UUID nodeId : nodeIds) {
- int port = portMgr.acquirePort(nodeId);
- spec.addTask(WORKER_JOB_NAME, nodeId, port);
- }
- }
-
- /**
- * Resolves TensorFlow cluster chief job and acquires ports.
- *
- * @param spec TensorFlow cluster specification.
- */
- private void resolveAndAcquirePortsForChief(TensorFlowClusterSpec spec) {
- ClusterNode chiefNode = ignite.cluster().localNode();
- UUID chiefNodeId = chiefNode.id();
- int chiefPort = portMgr.acquirePort(chiefNodeId);
-
- spec.addTask(CHIEF_JOB_NAME, chiefNodeId, chiefPort);
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowProcessBuilderSupplier.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowProcessBuilderSupplier.java
deleted file mode 100644
index 73f4b4b..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowProcessBuilderSupplier.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster.util;
-
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.internal.processors.odbc.ClientListenerProcessor;
-import org.apache.ignite.tensorflow.core.util.PythonProcessBuilderSupplier;
-
-/**
- * Python process builder supplier that is used to create TensorFlow worker process builder.
- */
-public class TensorFlowProcessBuilderSupplier extends PythonProcessBuilderSupplier {
- /** */
- private static final long serialVersionUID = 6866243505446122897L;
-
- /** Prefix for worker environment variables. */
- private static final String ENV_PREFIX = "IGNITE_DATASET_";
-
- /** Upstream cache query local mode. */
- private final Boolean loc;
-
- /**
- * Constructs a new instance of Python process builder supplier.
- *
- * @param interactive Interactive flag (allows to used standard input to pass Python script).
- * @param loc Upstream cache query local mode.
- * @param meta Meta information that adds to script as arguments.
- */
- public TensorFlowProcessBuilderSupplier(boolean interactive, Boolean loc, String... meta) {
- super(interactive, meta);
- this.loc = loc;
- }
-
- /** {@inheritDoc} */
- @Override public ProcessBuilder get() {
- ProcessBuilder pythonProcBuilder = super.get();
-
- Ignite ignite = Ignition.ignite();
- ClusterNode locNode = ignite.cluster().localNode();
-
- Integer port = locNode.attribute(ClientListenerProcessor.CLIENT_LISTENER_PORT);
-
- Map<String, String> env = pythonProcBuilder.environment();
- env.put(ENV_PREFIX + "HOST", "localhost");
-
- if (port != null)
- env.put(ENV_PREFIX + "PORT", String.valueOf(port));
-
- if (loc != null)
- env.put(ENV_PREFIX + "LOCAL", String.valueOf(loc));
-
- return pythonProcBuilder;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowUserScriptRunner.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowUserScriptRunner.java
deleted file mode 100644
index d9ed9b2..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowUserScriptRunner.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster.util;
-
-import java.io.BufferedOutputStream;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.util.List;
-import java.util.Map;
-import java.util.StringJoiner;
-import java.util.concurrent.ExecutorService;
-import java.util.function.Consumer;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-import org.apache.commons.io.IOUtils;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.tensorflow.cluster.TensorFlowJobArchive;
-import org.apache.ignite.tensorflow.cluster.spec.TensorFlowClusterSpec;
-import org.apache.ignite.tensorflow.cluster.spec.TensorFlowServerAddressSpec;
-import org.apache.ignite.tensorflow.core.util.AsyncNativeProcessRunner;
-import org.apache.ignite.tensorflow.core.util.NativeProcessRunner;
-
-/**
- * Utils class that helps to start and stop user script process.
- */
-public class TensorFlowUserScriptRunner extends AsyncNativeProcessRunner {
- /** Ignite logger. */
- private final IgniteLogger log;
-
- /** Job archive that will be extracted and used as working directory for the native process. */
- private final TensorFlowJobArchive jobArchive;
-
- /** TensorFlow cluster specification. */
- private final TensorFlowClusterSpec clusterSpec;
-
- /** Output stream data consumer. */
- private final Consumer<String> out;
-
- /** Error stream data consumer. */
- private final Consumer<String> err;
-
- /** Working directory of the user script process. */
- private File workingDir;
-
- /**
- * Constructs a new instance of TensorFlow user script runner.
- *
- * @param ignite Ignite instance.
- * @param executor Executor to be used in {@link AsyncNativeProcessRunner}.
- * @param jobArchive Job archive that will be extracted and used as working directory for the native process.
- * @param clusterSpec TensorFlow cluster specification.
- * @param out Output stream data consumer.
- * @param err Error stream data consumer.
- */
- public TensorFlowUserScriptRunner(Ignite ignite, ExecutorService executor, TensorFlowJobArchive jobArchive,
- TensorFlowClusterSpec clusterSpec, Consumer<String> out, Consumer<String> err) {
- super(ignite, executor);
-
- this.log = ignite.log().getLogger(TensorFlowUserScriptRunner.class);
-
- this.jobArchive = jobArchive;
- this.clusterSpec = clusterSpec;
- this.out = out;
- this.err = err;
- }
-
- /** {@inheritDoc} */
- @Override public NativeProcessRunner doBefore() {
- try {
- workingDir = Files.createTempDirectory("tf_us_").toFile();
- log.debug("Directory has been created [path=" + workingDir.getAbsolutePath() + "]");
-
- unzip(jobArchive.getData(), workingDir);
- log.debug("Job archive has been extracted [path=" + workingDir.getAbsolutePath() + "]");
-
- return prepareNativeProcessRunner();
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void doAfter() {
- if (workingDir != null) {
- delete(workingDir);
- log.debug("Directory has been deleted [path=" + workingDir.getAbsolutePath() + "]");
- }
- }
-
- /**
- * Prepares process builder and specifies working directory and command to be run.
- *
- * @return Prepared process builder.
- */
- private NativeProcessRunner prepareNativeProcessRunner() {
- if (workingDir == null)
- throw new IllegalStateException("Working directory is not created");
-
- ProcessBuilder procBuilder = new TensorFlowProcessBuilderSupplier(false, null).get();
-
- procBuilder.directory(workingDir);
- procBuilder.command(jobArchive.getCommands());
-
- Map<String, String> env = procBuilder.environment();
- env.put("PYTHONPATH", workingDir.getAbsolutePath());
- env.put("TF_CLUSTER", formatTfClusterVar());
- env.put("TF_WORKERS", formatTfWorkersVar());
- env.put("TF_CHIEF_SERVER", formatTfChiefServerVar());
-
- return new NativeProcessRunner(procBuilder, null, out, err);
- }
-
- /**
- * Formats "TF_CLUSTER" variable to be passed into user script.
- *
- * @return Formatted "TF_CLUSTER" variable to be passed into user script.
- */
- private String formatTfClusterVar() {
- return clusterSpec.format(Ignition.ignite());
- }
-
- /**
- * Formats "TF_WORKERS" variable to be passed into user script.
- *
- * @return Formatted "TF_WORKERS" variable to be passed into user script.
- */
- private String formatTfWorkersVar() {
- StringJoiner joiner = new StringJoiner(", ");
-
- int cnt = clusterSpec.getJobs().get(TensorFlowClusterResolver.WORKER_JOB_NAME).size();
- for (int i = 0; i < cnt; i++)
- joiner.add("\"/job:" + TensorFlowClusterResolver.WORKER_JOB_NAME + "/task:" + i + "\"");
-
- return "[" + joiner + "]";
- }
-
- /**
- * Formats "TF_CHIEF_SERVER" variable to be passed into user script.
- *
- * @return Formatted "TF_CHIEF_SERVER" variable to be passed into user script.
- */
- private String formatTfChiefServerVar() {
- List<TensorFlowServerAddressSpec> tasks = clusterSpec.getJobs().get(TensorFlowClusterResolver.CHIEF_JOB_NAME);
-
- if (tasks == null || tasks.size() != 1)
- throw new IllegalStateException("TensorFlow cluster specification should contain exactly one chief task");
-
- TensorFlowServerAddressSpec addrSpec = tasks.iterator().next();
-
- return "grpc://" + addrSpec.format(Ignition.ignite());
- }
-
- /**
- * Clears given file or directory recursively.
- *
- * @param file File or directory to be cleaned,
- */
- private void delete(File file) {
- if (file.isDirectory()) {
- String[] files = file.list();
-
- if (files != null && files.length != 0)
- for (String fileToBeDeleted : files)
- delete(new File(file, fileToBeDeleted));
-
- if (!file.delete())
- throw new IllegalStateException("Can't delete directory [path=" + file.getAbsolutePath() + "]");
- }
- else {
- if (!file.delete())
- throw new IllegalStateException("Can't delete file [path=" + file.getAbsolutePath() + "]");
- }
- }
-
- /**
- * Extracts specified zip archive into specified directory.
- *
- * @param data Zip archive to be extracted.
- * @param extractTo Target directory.
- */
- private void unzip(byte[] data, File extractTo) {
- try (ZipInputStream zipStream = new ZipInputStream(new ByteArrayInputStream(data))) {
- ZipEntry entry;
- while ((entry = zipStream.getNextEntry()) != null) {
- File file = new File(extractTo, entry.getName());
-
- if (entry.isDirectory() && !file.exists()) {
- boolean created = file.mkdirs();
- if (!created)
- throw new IllegalStateException("Can't create directory [path=" + file.getAbsolutePath() + "]");
- }
- else {
- if (!file.getParentFile().exists()) {
- boolean created = file.getParentFile().mkdirs();
- if (!created)
- throw new IllegalStateException("Can't create directory [path=" +
- file.getParentFile().getAbsolutePath() + "]");
- }
-
- try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(file))) {
- IOUtils.copy(zipStream, out);
- }
- }
- }
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/package-info.java
deleted file mode 100644
index f48d20b..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Util components that are used in TensorFlow cluster package.
- */
-
-package org.apache.ignite.tensorflow.cluster.util;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManager.java
deleted file mode 100644
index c825448..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManager.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core;
-
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
-
-/**
- * Process manager that allows to run and maintain processes in the cluster.
- *
- * @param <R> Type of task to be run.
- */
-public interface ProcessManager<R> {
- /**
- * Starts the processes by the given specifications.
- *
- * @param specifications Process specifications.
- * @return Map of node identifier as a key and list of started process identifiers as a value.
- */
- public Map<UUID, List<UUID>> start(List<R> specifications);
-
- /**
- * Pings the given processes.
- *
- * @param procIds Map of node identifier as a key and list of process identifiers as a value.
- * @return Map of node identifier as a key and list of process statuses as a value.
- */
- public Map<UUID, List<LongRunningProcessStatus>> ping(Map<UUID, List<UUID>> procIds);
-
- /**
- * Stops the given processes.
- *
- * @param procIds Map of node identifier as a key and list of process identifiers as a value.
- * @return Map of node identifier as a key and list of process statuses as a value.
- */
- public Map<UUID, List<LongRunningProcessStatus>> stop(Map<UUID, List<UUID>> procIds, boolean clear);
-
- /**
- * Clears metadata of the given processes.
- *
- * @param procIds Map of node identifier as a key and list of process identifiers as a value.
- * @return Map of node identifier as a key and list of process statuses as a value.
- */
- public Map<UUID, List<LongRunningProcessStatus>> clear(Map<UUID, List<UUID>> procIds);
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManagerWrapper.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManagerWrapper.java
deleted file mode 100644
index 4f10e83..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManagerWrapper.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
-
-/**
- * Process manager wrapper that allows to define how one type of process specification should be transformed into
- * another type of process specification delegate working with and delegate all operations to this delegate.
- *
- * @param <T> Type of process specification delegate working with.
- * @param <R> Type of accepted process specifications.
- */
-public abstract class ProcessManagerWrapper<T, R> implements ProcessManager<R> {
- /** Delegate. */
- private final ProcessManager<T> delegate;
-
- /**
- * Constructs a new instance of process manager wrapper.
- *
- * @param delegate Delegate.
- */
- public ProcessManagerWrapper(ProcessManager<T> delegate) {
- assert delegate != null : "Delegate should not be null";
-
- this.delegate = delegate;
- }
-
- /**
- * Transforms accepted process specification into process specification delegate working with.
- *
- * @param spec Accepted process specification.
- * @return Process specification delegate working with.
- */
- protected abstract T transformSpecification(R spec);
-
- /** {@inheritDoc} */
- @Override public Map<UUID, List<UUID>> start(List<R> specifications) {
- List<T> transformedSpecifications = new ArrayList<>();
-
- for (R spec : specifications)
- transformedSpecifications.add(transformSpecification(spec));
-
- return delegate.start(transformedSpecifications);
- }
-
- /** {@inheritDoc} */
- @Override public Map<UUID, List<LongRunningProcessStatus>> ping(Map<UUID, List<UUID>> procIds) {
- return delegate.ping(procIds);
- }
-
- /** {@inheritDoc} */
- @Override public Map<UUID, List<LongRunningProcessStatus>> stop(Map<UUID, List<UUID>> procIds, boolean clear) {
- return delegate.stop(procIds, clear);
- }
-
- /** {@inheritDoc} */
- @Override public Map<UUID, List<LongRunningProcessStatus>> clear(Map<UUID, List<UUID>> procIds) {
- return delegate.clear(procIds);
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcess.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcess.java
deleted file mode 100644
index f11f42a..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcess.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning;
-
-import java.io.Serializable;
-import java.util.UUID;
-import org.apache.ignite.lang.IgniteRunnable;
-
-/**
- * Long running process specification that contains identifier of a node where the process should be running on and the
- * task to be run.
- */
-public class LongRunningProcess implements Serializable {
- /** */
- private static final long serialVersionUID = 6039507725567997183L;
-
- /** Node identifier. */
- private final UUID nodeId;
-
- /** Task to be run. */
- private final IgniteRunnable task;
-
- /**
- * Constructs a new instance of long running process specification.
- *
- * @param nodeId Node identifier.
- * @param task Task to be run.
- */
- public LongRunningProcess(UUID nodeId, IgniteRunnable task) {
- assert nodeId != null : "Node identifier should not be null";
- assert task != null : "Task should not be null";
-
- this.nodeId = nodeId;
- this.task = task;
- }
-
- /** */
- public UUID getNodeId() {
- return nodeId;
- }
-
- /** */
- public IgniteRunnable getTask() {
- return task;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManager.java
deleted file mode 100644
index a25ff97..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManager.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.cluster.ClusterGroup;
-import org.apache.ignite.cluster.ClusterGroupEmptyException;
-import org.apache.ignite.tensorflow.core.ProcessManager;
-import org.apache.ignite.tensorflow.core.longrunning.task.LongRunningProcessClearTask;
-import org.apache.ignite.tensorflow.core.longrunning.task.LongRunningProcessPingTask;
-import org.apache.ignite.tensorflow.core.longrunning.task.LongRunningProcessStartTask;
-import org.apache.ignite.tensorflow.core.longrunning.task.LongRunningProcessStopTask;
-import org.apache.ignite.tensorflow.core.longrunning.task.LongRunningProcessTask;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
-
-/**
- * Long running process manager that allows to start, stop and make other actions with long running processes.
- */
-public class LongRunningProcessManager implements ProcessManager<LongRunningProcess> {
- /** Ignite instance. */
- private final Ignite ignite;
-
- /**
- * Constructs a new instance of long running process manager.
- *
- * @param ignite Ignite instance.
- */
- public LongRunningProcessManager(Ignite ignite) {
- assert ignite != null : "Ignite instance should not be null";
-
- this.ignite = ignite;
- }
-
- /** {@inheritDoc} */
- @Override public Map<UUID, List<UUID>> start(List<LongRunningProcess> specifications) {
- return call(groupByNodeId(specifications), LongRunningProcessStartTask::new, this::rollbackStartTask, false);
- }
-
- /** {@inheritDoc} */
- @Override public Map<UUID, List<LongRunningProcessStatus>> ping(Map<UUID, List<UUID>> procIds) {
- return call(procIds, LongRunningProcessPingTask::new, this::rollbackNothing, false);
- }
-
- /** {@inheritDoc} */
- @Override public Map<UUID, List<LongRunningProcessStatus>> stop(Map<UUID, List<UUID>> procIds, boolean clear) {
- return call(procIds, params -> new LongRunningProcessStopTask(params, clear), this::rollbackNothing, true);
- }
-
- /** {@inheritDoc} */
- @Override public Map<UUID, List<LongRunningProcessStatus>> clear(Map<UUID, List<UUID>> procIds) {
- return call(procIds, LongRunningProcessClearTask::new, this::rollbackNothing, true);
- }
-
- /**
- * Sends the specified tasks to the cluster to be executed.
- *
- * @param params Parameters needed to create tasks.
- * @param taskSupplier Supplier that defines how to create tasks.
- * @param rollback Rollback procedure.
- * @param onlyIfNodeExists If node doesn't exist the correspondent task will be ignored.
- * @param <T> Type of params needed to create tasks.
- * @param <E> Type of returned by task value.
- * @return Map of node identifier as a key and list task results as a value.
- */
- private <T, E> Map<UUID, List<E>> call(Map<UUID, List<T>> params,
- Function<List<T>, LongRunningProcessTask<List<E>>> taskSupplier, Consumer<Map<UUID, List<E>>> rollback,
- boolean onlyIfNodeExists) {
- Map<UUID, List<E>> res = new HashMap<>();
-
- try {
- for (UUID nodeId : params.keySet()) {
- List<T> nodeProcesses = params.get(nodeId);
- LongRunningProcessTask<List<E>> task = taskSupplier.apply(nodeProcesses);
-
- ClusterGroup clusterGrp = ignite.cluster().forNodeId(nodeId);
-
- try {
- List<E> nodeRes = ignite.compute(clusterGrp).call(task);
- res.put(nodeId, nodeRes);
- }
- catch (ClusterGroupEmptyException e) {
- if (!onlyIfNodeExists)
- throw e;
- }
- }
- }
- catch (Exception e) {
- // All-or-nothing strategy. In case of exception already processed tasks should be rolled back.
- rollback.accept(res);
-
- throw e;
- }
-
- return res;
- }
-
- /**
- * Groups the given process specifications by node identifier.
- *
- * @param specifications Process specifications.
- * @return Map of node identifier as a key and list of process specifications as a value.
- */
- private Map<UUID, List<LongRunningProcess>> groupByNodeId(List<LongRunningProcess> specifications) {
- Map<UUID, List<LongRunningProcess>> res = new HashMap<>();
-
- for (LongRunningProcess spec : specifications) {
- UUID nodeId = spec.getNodeId();
-
- List<LongRunningProcess> nodeSpecs = res.get(nodeId);
-
- if (nodeSpecs == null) {
- nodeSpecs = new ArrayList<>();
- nodeSpecs.add(spec);
- res.put(nodeId, nodeSpecs);
- }
- else
- nodeSpecs.add(spec);
- }
-
- return res;
- }
-
- /**
- * Rolls back start task successfully applied earlier.
- *
- * @param procIds Process identifiers.
- */
- private void rollbackStartTask(Map<UUID, List<UUID>> procIds) {
- stop(procIds, true);
- }
-
- /**
- * Rolls back nothing. Ping, stop and clear tasks cannot be rolled back, so it's the only one available strategy
- * for these tasks.
- */
- private void rollbackNothing(Map<UUID, List<LongRunningProcessStatus>> processes) {
- // Do nothing.
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/package-info.java
deleted file mode 100644
index d1edf16..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/package-info.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * The part of TensorFlow integration infrastructure that allows to start and maintain abstract long-running processes.
- * As described in {@link org.apache.ignite.tensorflow.core.longrunning.LongRunningProcess} user only needs to specify
- * runnable task and the node identifier the process should be started on and LongRunning Process Manager will make the
- * rest so that the specified runnable will be executed and maintained on the specified node.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessClearTask.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessClearTask.java
deleted file mode 100644
index 2afaaa9..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessClearTask.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning.task;
-
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessState;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.Future;
-
-/**
- * Task that clears process metadata on the node where the process has run.
- */
-public class LongRunningProcessClearTask extends LongRunningProcessTask<List<LongRunningProcessStatus>> {
- /** */
- private static final long serialVersionUID = -1840332865137076107L;
-
- /** Process identifiers. */
- private final List<UUID> procIds;
-
- /**
- * Constructs a new instance of clear task.
- *
- * @param procIds Process identifiers.
- */
- public LongRunningProcessClearTask(List<UUID> procIds) {
- assert procIds != null : "Process identifiers should not be null";
-
- this.procIds = procIds;
- }
-
- /** {@inheritDoc} */
- @Override public List<LongRunningProcessStatus> call() {
- ArrayList<LongRunningProcessStatus> res = new ArrayList<>();
-
- for (UUID prodId : procIds)
- res.add(prepareProcessForRemoving(prodId));
-
- // All-or-nothing strategy. Processes will be removed only if all processes can be removed.
- removeProcessesFromMetadataStorage();
-
- return res;
- }
-
- /**
- * Prepares process to be removed. Checks that the process is not running and wraps information about state and
- * exception into a status object.
- *
- * @param procId Process identifier.
- * @return Process status.
- */
- private LongRunningProcessStatus prepareProcessForRemoving(UUID procId) {
- Map<UUID, Future<?>> metadataStorage = getMetadataStorage();
-
- Future<?> fut = metadataStorage.get(procId);
-
- if (fut == null)
- return new LongRunningProcessStatus(LongRunningProcessState.NOT_FOUND);
-
- if (!fut.isDone())
- throw new IllegalStateException("Process is still running [procId=" + procId + "]");
-
- try {
- fut.get();
- return new LongRunningProcessStatus(LongRunningProcessState.DONE);
- }
- catch (Exception e) {
- return new LongRunningProcessStatus(LongRunningProcessState.DONE, e);
- }
- }
-
- /**
- * Removes processes from metadata storage.
- */
- private void removeProcessesFromMetadataStorage() {
- Map<UUID, Future<?>> metadataStorage = getMetadataStorage();
-
- for (UUID procId : procIds)
- metadataStorage.remove(procId);
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessPingTask.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessPingTask.java
deleted file mode 100644
index 5d54886..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessPingTask.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning.task;
-
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessState;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.Future;
-
-/**
- * Task that pings processes to check their statuses.
- */
-public class LongRunningProcessPingTask extends LongRunningProcessTask<List<LongRunningProcessStatus>> {
- /** */
- private static final long serialVersionUID = 7003289989579770395L;
-
- /** Process identifiers. */
- private final List<UUID> procIds;
-
- /**
- * Constructs a new instance of ping task.
- *
- * @param procIds Process identifiers.
- */
- public LongRunningProcessPingTask(List<UUID> procIds) {
- assert procIds != null : "Process identifiers should not be null";
-
- this.procIds = procIds;
- }
-
- /** {@inheritDoc} */
- @Override public List<LongRunningProcessStatus> call() {
- ArrayList<LongRunningProcessStatus> statuses = new ArrayList<>();
-
- for (UUID procId : procIds)
- statuses.add(getProcessStatus(procId));
-
- return statuses;
- }
-
- /**
- * Extracts the process status.
- *
- * @param procId Process identifier.
- * @return Process status.
- */
- private LongRunningProcessStatus getProcessStatus(UUID procId) {
- Map<UUID, Future<?>> metadataStorage = getMetadataStorage();
-
- Future<?> fut = metadataStorage.get(procId);
-
- if (fut == null)
- return new LongRunningProcessStatus(LongRunningProcessState.NOT_FOUND);
-
- if (!fut.isDone())
- return new LongRunningProcessStatus(LongRunningProcessState.RUNNING);
-
- try {
- fut.get();
- }
- catch (Exception e) {
- return new LongRunningProcessStatus(LongRunningProcessState.DONE, e);
- }
-
- return new LongRunningProcessStatus(LongRunningProcessState.DONE);
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStartTask.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStartTask.java
deleted file mode 100644
index 04f90d3..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStartTask.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning.task;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import org.apache.ignite.tensorflow.core.longrunning.LongRunningProcess;
-import org.apache.ignite.tensorflow.core.util.CustomizableThreadFactory;
-
-/**
- * Task that starts long running processes by their specifications.
- */
-public class LongRunningProcessStartTask extends LongRunningProcessTask<List<UUID>> {
- /** */
- private static final long serialVersionUID = -3934183044853083034L;
-
- /** Process specifications. */
- private final List<LongRunningProcess> processes;
-
- /**
- * Constructs a new instance of start task.
- *
- * @param processes Process specifications.
- */
- public LongRunningProcessStartTask(List<LongRunningProcess> processes) {
- assert processes != null : "Processes should not be null";
-
- this.processes = processes;
- }
-
- /** {@inheritDoc} */
- @Override public List<UUID> call() {
- ArrayList<UUID> res = new ArrayList<>();
-
- try {
- for (LongRunningProcess proc : processes) {
- Future<?> fut = runTask(proc.getTask());
-
- UUID procId = saveProcMetadata(fut);
-
- res.add(procId);
- }
- }
- catch (Exception e) {
- // All-or-nothing strategy. In case of exception already started processes will be stopped.
- stopAllProcessesAndClearMetadata(res);
-
- throw e;
- }
-
- return res;
- }
-
- /**
- * Executes the task in a separate thread.
- *
- * @param task Task to be executed.
- * @return Future that allows to interrupt or get the status of the task.
- */
- private Future<?> runTask(Runnable task) {
- return Executors
- .newSingleThreadExecutor(new CustomizableThreadFactory("tf-long-running", true))
- .submit(task);
- }
-
- /**
- * Saves process metadata into the local metadata storage.
- *
- * @param fut Future that allows to interrupt or get the status of the task.
- * @return Process identifier.
- */
- private UUID saveProcMetadata(Future<?> fut) {
- Map<UUID, Future<?>> metadataStorage = getMetadataStorage();
-
- UUID procId = UUID.randomUUID();
-
- metadataStorage.put(procId, fut);
-
- return procId;
- }
-
- /**
- * Stop all processes by their identifiers and removes them from the metadata storage.
- *
- * @param procIds Process identifiers.
- */
- private void stopAllProcessesAndClearMetadata(List<UUID> procIds) {
- Map<UUID, Future<?>> metadataStorage = getMetadataStorage();
-
- for (UUID procId : procIds) {
- Future<?> fut = metadataStorage.remove(procId);
- fut.cancel(true);
- }
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStopTask.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStopTask.java
deleted file mode 100644
index 56b906b..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStopTask.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning.task;
-
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessState;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.Future;
-
-/**
- * Task that stops long running processes.
- */
-public class LongRunningProcessStopTask extends LongRunningProcessTask<List<LongRunningProcessStatus>> {
- /** */
- private static final long serialVersionUID = -5552468435820611170L;
-
- /** Process identifiers. */
- private final List<UUID> procIds;
-
- /** Flag that defines that metadata should be removed immediately. */
- private final boolean clear;
-
- /**
- * Constructs a new instance of stop task.
- *
- * @param procIds Process identifiers.
- * @param clear Flag that defines that metadata should be removed immediately.
- */
- public LongRunningProcessStopTask(List<UUID> procIds, boolean clear) {
- assert procIds != null : "Process identifiers should not be null";
-
- this.procIds = procIds;
- this.clear = clear;
- }
-
- /** {@inheritDoc} */
- @Override public List<LongRunningProcessStatus> call() {
- ArrayList<LongRunningProcessStatus> res = new ArrayList<>();
-
- for (UUID prodId : procIds)
- res.add(stopProcess(prodId));
-
- // All-or-nothing strategy. Processes will be removed only if all processes can be removed.
- if (clear)
- removeProcessesFromMetadataStorage();
-
- return res;
- }
-
- /**
- * Stop process by process identifier.
- *
- * @param procId Process identifier.
- * @return Process status after stop.
- */
- private LongRunningProcessStatus stopProcess(UUID procId) {
- Map<UUID, Future<?>> metadataStorage = getMetadataStorage();
-
- Future<?> fut = metadataStorage.get(procId);
-
- if (fut == null)
- return new LongRunningProcessStatus(LongRunningProcessState.NOT_FOUND);
-
- try {
- fut.cancel(true);
- fut.get();
- }
- catch (Exception e) {
- return new LongRunningProcessStatus(LongRunningProcessState.DONE, e);
- }
-
- return new LongRunningProcessStatus(LongRunningProcessState.DONE);
- }
-
- /**
- * Removes processes from metadata storage.
- */
- private void removeProcessesFromMetadataStorage() {
- Map<UUID, Future<?>> metadataStorage = getMetadataStorage();
-
- for (UUID procId : procIds)
- metadataStorage.remove(procId);
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessTask.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessTask.java
deleted file mode 100644
index 96d2274..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessTask.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning.task;
-
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.Future;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.lang.IgniteCallable;
-import org.apache.ignite.tensorflow.core.longrunning.LongRunningProcessManager;
-
-/**
- * Task that can be executed on a cluster using the {@link LongRunningProcessManager}.
- *
- * @param <T> Type of the result.
- */
-public abstract class LongRunningProcessTask<T> implements IgniteCallable<T> {
- /** */
- private static final long serialVersionUID = 2351380200758417004L;
-
- /** Long running process storage name. */
- private static final String LONG_RUNNING_PROCESS_STORAGE_NAME = "LONG_RUNNING_PROCESS_STORAGE";
-
- /**
- * Returns task metadata storage that is kept by every node in the cluster.
- *
- * @return Task metadata storage that is kept by every node in the cluster.
- */
- ConcurrentHashMap<UUID, Future<?>> getMetadataStorage() {
- Ignite ignite = Ignition.localIgnite();
-
- ConcurrentMap<String, ConcurrentHashMap<UUID, Future<?>>> nodeLocMap = ignite.cluster().nodeLocalMap();
-
- return nodeLocMap.computeIfAbsent(LONG_RUNNING_PROCESS_STORAGE_NAME, k -> new ConcurrentHashMap<>());
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/package-info.java
deleted file mode 100644
index 34ffe72..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains long-running process tasks that encapsulates the logic of starting, pinging and stopping a long-running
- * process.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning.task;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/util/LongRunningProcessState.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/util/LongRunningProcessState.java
deleted file mode 100644
index 7b3c777..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/util/LongRunningProcessState.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning.task.util;
-
-/**
- * Long running process state.
- */
-public enum LongRunningProcessState {
- /** Process not found. */ NOT_FOUND,
- /** Process is running. */ RUNNING,
- /** Process is done */ DONE
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/util/LongRunningProcessStatus.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/util/LongRunningProcessStatus.java
deleted file mode 100644
index 1ae8e15..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/util/LongRunningProcessStatus.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning.task.util;
-
-import java.io.Serializable;
-
-/**
- * Long running process status that includes state and exception if exists.
- */
-public class LongRunningProcessStatus implements Serializable {
- /** */
- private static final long serialVersionUID = -2958316078722079954L;
-
- /** Process state. */
- private final LongRunningProcessState state;
-
- /** Process exception. */
- private final Exception e;
-
- /**
- * Constructs a new instance of long running process status.
- *
- * @param state Process state.
- */
- public LongRunningProcessStatus(LongRunningProcessState state) {
- this(state, null);
- }
-
- /**
- * Constructs a new instance of long running process status.
- *
- * @param state Process state.
- * @param e Process exception.
- */
- public LongRunningProcessStatus(LongRunningProcessState state, Exception e) {
- assert state != null : "Process state should not be null";
-
- this.state = state;
- this.e = e;
- }
-
- /** */
- public LongRunningProcessState getState() {
- return state;
- }
-
- /** */
- public Exception getException() {
- return e;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/util/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/util/package-info.java
deleted file mode 100644
index 645a8ef..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/util/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Util package that contains util classes used in long-running tasks.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning.task.util;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcess.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcess.java
deleted file mode 100644
index 2ad3c9d..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcess.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.nativerunning;
-
-import java.io.Serializable;
-import java.util.UUID;
-import java.util.function.Supplier;
-import org.apache.ignite.tensorflow.util.SerializableSupplier;
-
-/**
- * Native process specification.
- */
-public class NativeProcess implements Serializable {
- /** */
- private static final long serialVersionUID = -7056800139746134956L;
-
- /** Process builder supplier. */
- private final SerializableSupplier<ProcessBuilder> procBuilderSupplier;
-
- /** Stdin of the process. */
- private final String stdin;
-
- /** Node identifier. */
- private final UUID nodeId;
-
- /**
- * Constructs a new instance of native process specification.
- *
- * @param procBuilderSupplier Process builder supplier.
- * @param stdin Stdin of the process.
- * @param nodeId Node identifier.
- */
- public NativeProcess(SerializableSupplier<ProcessBuilder> procBuilderSupplier, String stdin, UUID nodeId) {
- assert procBuilderSupplier != null : "Process builder supplier should not be null";
- assert nodeId != null : "Node identifier should not be null";
-
- this.procBuilderSupplier = procBuilderSupplier;
- this.stdin = stdin;
- this.nodeId = nodeId;
- }
-
- /** */
- public Supplier<ProcessBuilder> getProcBuilderSupplier() {
- return procBuilderSupplier;
- }
-
- /** */
- public String getStdin() {
- return stdin;
- }
-
- /** */
- public UUID getNodeId() {
- return nodeId;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcessManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcessManager.java
deleted file mode 100644
index 5accf3d..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcessManager.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.nativerunning;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.tensorflow.core.ProcessManager;
-import org.apache.ignite.tensorflow.core.ProcessManagerWrapper;
-import org.apache.ignite.tensorflow.core.longrunning.LongRunningProcess;
-import org.apache.ignite.tensorflow.core.longrunning.LongRunningProcessManager;
-import org.apache.ignite.tensorflow.core.nativerunning.task.NativeProcessStartTask;
-
-/**
- * Native process manager that allows to start, stop and make other actions with native processes.
- */
-public class NativeProcessManager extends ProcessManagerWrapper<LongRunningProcess, NativeProcess> {
- /**
- * Constructs a new native process manager.
- *
- * @param ignite Ignite instance.
- */
- public NativeProcessManager(Ignite ignite) {
- super(new LongRunningProcessManager(ignite));
- }
-
- /**
- * Constructs a new native process manager.
- *
- * @param delegate Delegate.
- */
- public NativeProcessManager(ProcessManager<LongRunningProcess> delegate) {
- super(delegate);
- }
-
- /** {@inheritDoc} */
- @Override protected LongRunningProcess transformSpecification(NativeProcess spec) {
- return new LongRunningProcess(spec.getNodeId(), new NativeProcessStartTask(spec));
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/package-info.java
deleted file mode 100644
index 86b64b3..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/package-info.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * The part of TensorFlow integration infrastructure that allows to start and maintain abstract native processes. As
- * described in {@link org.apache.ignite.tensorflow.core.nativerunning.NativeProcess} user only needs to specify process
- * builder, stdin and node identifier the process should be started on and Native Process Manager will make the rest so
- * that the specified process will be executed and maintained on the specified node.
- */
-
-package org.apache.ignite.tensorflow.core.nativerunning;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/task/NativeProcessStartTask.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/task/NativeProcessStartTask.java
deleted file mode 100644
index ae9e2b9..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/task/NativeProcessStartTask.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.nativerunning.task;
-
-import java.util.function.Supplier;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.lang.IgniteRunnable;
-import org.apache.ignite.tensorflow.core.nativerunning.NativeProcess;
-import org.apache.ignite.tensorflow.core.util.NativeProcessRunner;
-
-/**
- * Task that starts native process by its specification.
- */
-public class NativeProcessStartTask implements IgniteRunnable {
- /** */
- private static final long serialVersionUID = 8421398298283116405L;
-
- /** Native process specification. */
- private final NativeProcess procSpec;
-
- /**
- * Constructs a new instance of native process start task.
- *
- * @param procSpec Native process specification.
- */
- public NativeProcessStartTask(NativeProcess procSpec) {
- assert procSpec != null : "Process specification should not be null";
-
- this.procSpec = procSpec;
- }
-
- /** {@inheritDoc} */
- @Override public void run() {
- Supplier<ProcessBuilder> procBuilderSupplier = procSpec.getProcBuilderSupplier();
- ProcessBuilder procBuilder = procBuilderSupplier.get();
-
- NativeProcessRunner procRunner = new NativeProcessRunner(
- procBuilder,
- procSpec.getStdin(),
- System.out::println,
- System.err::println
- );
-
- IgniteLogger log = Ignition.ignite().log().getLogger(NativeProcessStartTask.class);
-
- try {
- log.debug("Starting native process");
- procRunner.startAndWait();
- log.debug("Native process completed");
- }
- catch (InterruptedException e) {
- log.debug("Native process interrupted");
- }
- catch (Exception e) {
- log.error("Native process failed", e);
- throw e;
- }
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/task/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/task/package-info.java
deleted file mode 100644
index 02222fc..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/task/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains native process task that encapsulates the logic of starting native process.
- */
-
-package org.apache.ignite.tensorflow.core.nativerunning.task;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/package-info.java
deleted file mode 100644
index dc8b314..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/package-info.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * TensorFlow integration core package that provides infrastructure layers that allows TensorFlow cluster to start and
- * be maintained. It provides layer hierarchy. The lowermost layer (long-running process layer) provides API to start
- * and maintain abstract long-running processes. The second layer (native-running processes layer) is built on top of
- * previous layer and allows to start and maintain native processes. And the third layer (python-running processes
- * layer) is responsible for starting and maintaining of Python native processes.
- */
-
-package org.apache.ignite.tensorflow.core;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/AsyncNativeProcessRunner.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/AsyncNativeProcessRunner.java
deleted file mode 100644
index 8bf32e8..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/AsyncNativeProcessRunner.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.util;
-
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteLogger;
-
-/**
- * Asynchronous native process runner.
- */
-public abstract class AsyncNativeProcessRunner {
- /** Ignite logger. */
- private final IgniteLogger log;
-
- /** Executors that is used to start async native process. */
- private final ExecutorService executor;
-
- /** Future of the async process process. */
- private Future<?> fut;
-
- /**
- * Constructs a new asynchronous native process runner.
- *
- * @param ignite Ignite instance.
- * @param executor Executor.
- */
- public AsyncNativeProcessRunner(Ignite ignite, ExecutorService executor) {
- this.log = ignite.log().getLogger(AsyncNativeProcessRunner.class);
- this.executor = executor;
- }
-
- /**
- * Method that should be called before starting the process.
- *
- * @return Prepared native process runner.
- */
- public abstract NativeProcessRunner doBefore();
-
- /**
- * Method that should be called after starting the process.
- */
- public abstract void doAfter();
-
- /**
- * Starts the process in separate thread.
- */
- public synchronized void start() {
- if (fut != null)
- throw new IllegalStateException("Async native process has already been started");
-
- NativeProcessRunner procRunner = doBefore();
-
- fut = executor.submit(() -> {
- try {
- log.debug("Starting native process");
- procRunner.startAndWait();
- log.debug("Native process completed");
- }
- catch (InterruptedException e) {
- log.debug("Native process interrupted");
- }
- catch (Exception e) {
- log.error("Native process failed", e);
- throw e;
- }
- finally {
- doAfter();
- }
- });
- }
-
- /**
- * Stops the process.
- */
- public synchronized void stop() {
- if (fut != null && !fut.isDone())
- fut.cancel(true);
- }
-
- /**
- * Checks if process is already completed.
- *
- * @return {@code true} if process completed, otherwise {@code false}.
- */
- public boolean isCompleted() {
- return fut != null && fut.isDone();
- }
-
- /**
- * Returns an exception that happened during execution or {@code null} if there is no exception.
- *
- * @return Exception that happened during execution or {@code null} if there is no exception.
- */
- public Exception getException() {
- if (!fut.isDone())
- return null;
-
- try {
- fut.get();
- }
- catch (InterruptedException | ExecutionException e) {
- return e;
- }
-
- return null;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/CustomizableThreadFactory.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/CustomizableThreadFactory.java
deleted file mode 100644
index 2de671d..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/CustomizableThreadFactory.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.util;
-
-import java.util.concurrent.ThreadFactory;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * Customizable thread factory that allows to specify thread name and daemon flag for the processes created by this
- * factory.
- */
-public class CustomizableThreadFactory implements ThreadFactory {
- /** Thread name. */
- private final String threadName;
-
- /** Is daemon flag. */
- private final boolean isDaemon;
-
- /**
- * Constructs a new instance of customizable thread factory.
- *
- * @param threadName Thread name.
- * @param isDaemon Is daemon flag.
- */
- public CustomizableThreadFactory(String threadName, boolean isDaemon) {
- this.threadName = threadName;
- this.isDaemon = isDaemon;
- }
-
- /** {@inheritDoc} */
- @Override public Thread newThread(@NotNull Runnable r) {
- Thread thread = new Thread(r);
-
- thread.setName(threadName);
- thread.setDaemon(isDaemon);
-
- return thread;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/NativeProcessRunner.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/NativeProcessRunner.java
deleted file mode 100644
index 38af26d..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/NativeProcessRunner.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.util;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.util.Scanner;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Consumer;
-
-/**
- * Utils class that helps to start native processes.
- */
-public class NativeProcessRunner {
- /** Thread name to be used by threads that forward streams. */
- private static final String NATIVE_PROCESS_FORWARD_STREAM_THREAD_NAME = "tf-forward-native-output";
-
- /** Process builder. */
- private final ProcessBuilder procBuilder;
-
- /** Standard input of the process. */
- private final String stdin;
-
- /** Output stream data consumer. */
- private final Consumer<String> out;
-
- /** Error stream data consumer. */
- private final Consumer<String> err;
-
- /**
- * Constructs a new instance of native process runner.
- *
- * @param procBuilder Process builder.
- * @param stdin Standard input of the process.
- * @param out Output stream data consumer.
- * @param err Error stream data consumer.
- */
- public NativeProcessRunner(ProcessBuilder procBuilder, String stdin, Consumer<String> out, Consumer<String> err) {
- this.procBuilder = procBuilder;
- this.stdin = stdin;
- this.out = out;
- this.err = err;
- }
-
- /**
- * Starts the native process and waits it to be completed successfully or with exception.
- */
- public void startAndWait() throws InterruptedException {
- Process proc;
- try {
- proc = procBuilder.start();
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- AtomicBoolean shutdown = new AtomicBoolean();
-
- Thread shutdownHook = new Thread(() -> {
- shutdown.set(true);
- proc.destroy();
- });
-
- Runtime.getRuntime().addShutdownHook(shutdownHook);
-
- Future<?> outForward = forwardStream(proc.getInputStream(), out);
- Future<?> errForward = forwardStream(proc.getErrorStream(), err);
-
- try {
- if (stdin != null) {
- PrintWriter writer = new PrintWriter(proc.getOutputStream());
- writer.println(stdin);
- writer.flush();
- }
-
- int status;
- try {
- status = proc.waitFor();
- }
- catch (InterruptedException e) {
- proc.destroy();
- throw e;
- }
-
- if (!shutdown.get()) {
- Runtime.getRuntime().removeShutdownHook(shutdownHook);
-
- if (status != 0)
- throw new IllegalStateException("Native process exit [status=" + status + "]");
- }
- }
- finally {
- outForward.cancel(true);
- errForward.cancel(true);
- }
- }
-
- /**
- * Forwards stream.
- *
- * @param src Source stream.
- * @param dst Destination stream.
- * @return Future that allows to interrupt forwarding.
- */
- private Future<?> forwardStream(InputStream src, Consumer<String> dst) {
- return Executors
- .newSingleThreadExecutor(new CustomizableThreadFactory(NATIVE_PROCESS_FORWARD_STREAM_THREAD_NAME, true))
- .submit(() -> {
- Scanner scanner = new Scanner(src);
-
- while (!Thread.currentThread().isInterrupted() && scanner.hasNextLine())
- dst.accept(scanner.nextLine());
- });
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/PythonProcessBuilderSupplier.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/PythonProcessBuilderSupplier.java
deleted file mode 100644
index f3882e9..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/PythonProcessBuilderSupplier.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.util;
-
-import java.lang.management.ManagementFactory;
-import java.util.Map;
-import org.apache.ignite.tensorflow.util.SerializableSupplier;
-
-/**
- * Python process builder supplier that is used to create Python process builder.
- */
-public class PythonProcessBuilderSupplier implements SerializableSupplier<ProcessBuilder> {
- /** */
- private static final long serialVersionUID = 7181937306294456125L;
-
- /** Python environment variable name. */
- private static final String PYTHON_ENV_NAME = "PYTHON";
-
- /** Interactive flag (allows to used standard input to pass Python script). */
- private final boolean interactive;
-
- /** Meta information that adds to script as arguments. */
- private final String[] meta;
-
- /**
- * Constructs a new instance of Python process builder supplier.
- *
- * @param interactive Interactive flag (allows to used standard input to pass Python script).
- * @param meta Meta information that adds to script as arguments.
- */
- public PythonProcessBuilderSupplier(boolean interactive, String... meta) {
- this.interactive = interactive;
- this.meta = meta;
- }
-
- /** {@inheritDoc} */
- @Override public ProcessBuilder get() {
- String python = System.getenv(PYTHON_ENV_NAME);
-
- if (python == null)
- python = "python3";
-
- ProcessBuilder procBldr;
- if (interactive) {
- String[] cmd = new String[meta.length + 3];
-
- cmd[0] = python;
- cmd[1] = "-i";
- cmd[2] = "-";
-
- System.arraycopy(meta, 0, cmd, 3, meta.length);
-
- procBldr = new ProcessBuilder(cmd);
- }
- else
- procBldr = new ProcessBuilder(python);
-
- Map<String, String> env = procBldr.environment();
- env.put("PPID", String.valueOf(getProcessId()));
-
- return procBldr;
- }
-
- /**
- * Returns current process identifier.
- *
- * @return Process identifier.
- */
- private long getProcessId() {
- String pid = ManagementFactory.getRuntimeMXBean().getName().split("@")[0];
-
- return Long.parseLong(pid);
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/package-info.java
deleted file mode 100644
index 39119bd..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Util classes used in other {@link org.apache.ignite.tensorflow.core} classes.
- */
-
-package org.apache.ignite.tensorflow.core.util;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/package-info.java
deleted file mode 100644
index 53189de..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/package-info.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * TensorFlow integration that allows to start and maintain TensorFlow cluster on top of Apache Ignite cluster
- * infrastructure. The TensorFlow cluster is built for the specified cache the following way:
- * <ol>
- * <li>The TensorFlow cluster maintainer is created to maintain the cluster associated with the specified cache so
- * that this service works reliable even the node will fail. It's achieved using Ignite Service Grid.</li>
- * <li>TensorFlow cluster maintainer finds out that cluster is not started and begins starting procedure.</li>
- * <li>TensorFlow cluster resolver builds cluster specification based on the specified cache so that every
- * TensorFlow task is associated with partition of the cache and assumed to be started on the node where the
- * partition is kept.</li>
- * <li>Based on the built cluster specification the set of tasks is sent to the nodes on purpose to start TensorFlow
- * servers on the nodes defined in the specification.</li>
- * <li>When this set of tasks is completed successfully the started process identifiers are returned and saved
- * for future using.</li>
- * <li>The starting procedure is completed. In case a server fails the cluster will be turn down and then started
- * again by maintainer.</li>
- * </ol>
- */
-
-package org.apache.ignite.tensorflow;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/JobSubmitter.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/JobSubmitter.java
deleted file mode 100644
index 0a7cae6..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/JobSubmitter.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.submitter;
-
-import org.apache.ignite.tensorflow.submitter.command.RootCommand;
-import picocli.CommandLine;
-
-/**
- * Main class of the job submitter application that allows to submit TensorFlow jobs to be run within Ignite cluster.
- */
-public class JobSubmitter {
- /**
- * Main method.
- *
- * @param args Arguments.
- */
- public static void main(String... args) {
- CommandLine.run(new RootCommand(), System.out, args);
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/AbstractCommand.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/AbstractCommand.java
deleted file mode 100644
index 4d2fc18..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/AbstractCommand.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.submitter.command;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.logger.slf4j.Slf4jLogger;
-import picocli.CommandLine;
-
-/**
- * Abstract command that contains options common for all commands.
- */
-public abstract class AbstractCommand implements Runnable {
- /** Ignite node configuration path. */
- @CommandLine.Option(names = { "-c", "--config" }, description = "Apache Ignite client configuration.")
- protected String cfg;
-
- /**
- * Returns Ignite instance based on configuration specified in {@link #cfg} field.
- *
- * @return Ignite instance.
- */
- protected Ignite getIgnite() {
- if (cfg != null)
- return Ignition.start(cfg);
- else {
- IgniteConfiguration igniteCfg = new IgniteConfiguration();
- igniteCfg.setGridLogger(new Slf4jLogger());
- igniteCfg.setClientMode(true);
-
- return Ignition.start(igniteCfg);
- }
- }
-
- /** */
- public void setCfg(String cfg) {
- this.cfg = cfg;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/AttachCommand.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/AttachCommand.java
deleted file mode 100644
index 946aa08..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/AttachCommand.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.submitter.command;
-
-import java.util.UUID;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.tensorflow.cluster.TensorFlowClusterGatewayManager;
-import picocli.CommandLine;
-
-/**
- * Command "attach" that is used to attach to running TensorFlow cluster and receive output of the user script.
- */
-@CommandLine.Command(
- name = "attach",
- description = "Attaches to running TensorFlow cluster (user script process).",
- mixinStandardHelpOptions = true
-)
-public class AttachCommand extends AbstractCommand {
- /** TensorFlow cluster identifier. */
- @CommandLine.Parameters(paramLabel = "CLUSTER_ID", description = "Cluster identifier.")
- private UUID clusterId;
-
- /** {@inheritDoc} */
- @Override public void run() {
- try (Ignite ignite = getIgnite()) {
- TensorFlowClusterGatewayManager mgr = new TensorFlowClusterGatewayManager(ignite);
-
- mgr.listenToClusterUserScript(clusterId, System.out::println, System.err::println);
- }
- }
-
- /** */
- public void setClusterId(UUID clusterId) {
- this.clusterId = clusterId;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/PsCommand.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/PsCommand.java
deleted file mode 100644
index 0538496..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/PsCommand.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.submitter.command;
-
-import java.util.Map;
-import java.util.UUID;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.tensorflow.cluster.TensorFlowCluster;
-import org.apache.ignite.tensorflow.cluster.TensorFlowClusterManager;
-import picocli.CommandLine;
-
-/**
- * Command "ps" that is used to print identifiers of all running TensorFlow clusters.
- */
-@CommandLine.Command(
- name = "ps",
- description = "Prints identifiers of all running TensorFlow clusters.",
- mixinStandardHelpOptions = true
-)
-public class PsCommand extends AbstractCommand {
- /** {@inheritDoc} */
- @Override public void run() {
- try (Ignite ignite = getIgnite()) {
- TensorFlowClusterManager mgr = new TensorFlowClusterManager(ignite);
-
- Map<UUID, TensorFlowCluster> clusters = mgr.getAllClusters();
-
- for (UUID clusterId : clusters.keySet())
- System.out.println(clusterId);
- }
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/RootCommand.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/RootCommand.java
deleted file mode 100644
index 508ea7b..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/RootCommand.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.submitter.command;
-
-import picocli.CommandLine;
-
-/**
- * Root command that aggregates all sub commands.
- */
-@CommandLine.Command(
- name = "ignite-tf",
- description = "Apache Ignite and TensorFlow integration command line tool that allows to start, maintain and" +
- " stop distributed deep learning utilizing Apache Ignite infrastructure and data.",
- subcommands = {
- StartCommand.class,
- StopCommand.class,
- AttachCommand.class,
- PsCommand.class
- },
- mixinStandardHelpOptions = true
-)
-public class RootCommand extends AbstractCommand {
- /** {@inheritDoc} */
- @Override public void run() {
- CommandLine.usage(this, System.out);
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StartCommand.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StartCommand.java
deleted file mode 100644
index 0202b8e..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StartCommand.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.submitter.command;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.UUID;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
-import org.apache.commons.io.IOUtils;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
-import org.apache.ignite.tensorflow.cluster.TensorFlowClusterGatewayManager;
-import org.apache.ignite.tensorflow.cluster.TensorFlowJobArchive;
-import picocli.CommandLine;
-
-/**
- * Command "start" that is used to start a new TensorFlow cluster on top of Apache Ignite.
- */
-@CommandLine.Command(
- name = "start",
- description = "Starts a new TensorFlow cluster and attaches to user script process.",
- mixinStandardHelpOptions = true
-)
-public class StartCommand extends AbstractCommand {
- /** Upstream cache name. */
- @CommandLine.Parameters(index = "0", paramLabel = "CACHE_NAME", description = "Upstream cache name.")
- private String cacheName;
-
- /** Job folder or archive. */
- @CommandLine.Parameters(index = "1", paramLabel = "JOB_DIR", description = "Job folder (or zip archive).")
- private String jobFolder;
-
- /** Job command to be executed in cluster. */
- @CommandLine.Parameters(index = "2", paramLabel = "JOB_CMD", description = "Job command.")
- private String jobCmd;
-
- /** Arguments of a job command to be executed in cluster. */
- @CommandLine.Parameters(index = "3..*", paramLabel = "JOB_ARGS", description = "Job arguments.")
- private String[] jobArguments;
-
- /** {@inheritDoc} */
- @Override public void run() {
- try (Ignite ignite = getIgnite()) {
- UUID clusterId = UUID.randomUUID();
- String[] commands = new String[jobArguments.length + 1];
- commands[0] = jobCmd;
- System.arraycopy(jobArguments, 0, commands, 1, commands.length - 1);
-
- TensorFlowJobArchive jobArchive = new TensorFlowJobArchive(
- cacheName,
- zip(jobFolder),
- commands
- );
-
- TensorFlowClusterGatewayManager mgr = new TensorFlowClusterGatewayManager(ignite);
- mgr.createCluster(clusterId, jobArchive);
-
- mgr.listenToClusterUserScript(clusterId, System.out::println, System.err::println);
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Archives specified folder or file into zip archive.
- *
- * @param jobArchivePath Path to folder to be archived.
- * @return Byte array representing zip archive.
- * @throws IOException In case of input/output exception.
- */
- private byte[] zip(String jobArchivePath) throws IOException {
- Path path = Paths.get(jobArchivePath);
- File file = path.toFile();
-
- if (!file.exists())
- throw new IllegalArgumentException("File doesn't exist [name=" + jobArchivePath + "]");
-
- if (file.isDirectory())
- return zipDirectory(file);
- else if (jobArchivePath.endsWith(FilePageStoreManager.ZIP_SUFFIX))
- return zipArchive(file);
- else
- return zipFile(file);
- }
-
- /**
- * Archives specified folder into zip archive.
- *
- * @param dir Directory to be archived.
- * @return Byte array representing zip archive.
- * @throws IOException In case of input/output exception.
- */
- private byte[] zipDirectory(File dir) throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- try (ZipOutputStream zipFile = new ZipOutputStream(baos)) {
- compressDirectoryToZip(dir.getAbsolutePath(), dir.getAbsolutePath(), zipFile);
- }
-
- return baos.toByteArray();
- }
-
- /**
- * Archives specified file into zip archive.
- *
- * @param file File to be archived.
- * @return Byte array representing zip archive.
- * @throws IOException In case of input/output exception.
- */
- private byte[] zipFile(File file) throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- try (ZipOutputStream zos = new ZipOutputStream(baos)) {
- ZipEntry entry = new ZipEntry(file.getName());
- zos.putNextEntry(entry);
-
- try (FileInputStream in = new FileInputStream(file.getAbsolutePath())) {
- IOUtils.copy(in, zos);
- }
- }
-
- return baos.toByteArray();
- }
-
- /**
- * Reads zip archive into byte array and returns this array.
- *
- * @param file Archive to be read.
- * @return Byte array representing zip archive.
- * @throws IOException In case of input/output exception.
- */
- private byte[] zipArchive(File file) throws IOException {
- try (FileInputStream fis = new FileInputStream(file)) {
- return IOUtils.toByteArray(fis);
- }
- }
-
- /**
- * Archives specified folder into zip output stream.
- *
- * @param rootDir Root directory.
- * @param srcDir Source directory.
- * @param out Zip output stream.
- * @throws IOException In case of input/output exception.
- */
- private void compressDirectoryToZip(String rootDir, String srcDir, ZipOutputStream out) throws IOException {
- File[] files = new File(srcDir).listFiles();
-
- if (files != null) {
- for (File file : files) {
- if (file.isDirectory())
- compressDirectoryToZip(rootDir, srcDir + File.separator + file.getName(), out);
- else {
- ZipEntry entry = new ZipEntry(srcDir.replace(rootDir, "")
- + File.separator + file.getName());
- out.putNextEntry(entry);
-
- try (FileInputStream in = new FileInputStream(srcDir + File.separator + file.getName())) {
- IOUtils.copy(in, out);
- }
- }
- }
- }
- }
-
- /** */
- public void setCacheName(String cacheName) {
- this.cacheName = cacheName;
- }
-
- /** */
- public void setJobFolder(String jobFolder) {
- this.jobFolder = jobFolder;
- }
-
- /** */
- public void setJobCmd(String jobCmd) {
- this.jobCmd = jobCmd;
- }
-
- /** */
- public void setJobArguments(String[] jobArguments) {
- this.jobArguments = jobArguments;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StopCommand.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StopCommand.java
deleted file mode 100644
index 8890370..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StopCommand.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.submitter.command;
-
-import java.util.UUID;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.tensorflow.cluster.TensorFlowClusterGatewayManager;
-import picocli.CommandLine;
-
-/**
- * Command "stop" that is used to stop TensorFlow cluster.
- */
-@CommandLine.Command(
- name = "stop",
- description = "Stops a running TensorFlow cluster.",
- mixinStandardHelpOptions = true
-)
-public class StopCommand extends AbstractCommand {
- /** Cluster identifier. */
- @CommandLine.Parameters(paramLabel = "CLUSTER_ID", description = "Cluster identifier.")
- private UUID clusterId;
-
- /** {@inheritDoc} */
- @Override public void run() {
- try (Ignite ignite = getIgnite()) {
- TensorFlowClusterGatewayManager mgr = new TensorFlowClusterGatewayManager(ignite);
- mgr.stopClusterIfExists(clusterId);
- }
- }
-
- /** */
- public void setClusterId(UUID clusterId) {
- this.clusterId = clusterId;
- }
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/package-info.java
deleted file mode 100644
index 1693887..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * This package contains commands that command line tool provides. Pico CLI is used to make these commands maintainable.
- */
-
-package org.apache.ignite.tensorflow.submitter.command;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/package-info.java
deleted file mode 100644
index af6e27c..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * This package contains classes that allow to use command line interface to submit jobs into TensorFlow in Apache
- * Ignite infrastructure.
- */
-
-package org.apache.ignite.tensorflow.submitter;
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/SerializableSupplier.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/SerializableSupplier.java
deleted file mode 100644
index 768dbe1..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/SerializableSupplier.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.util;
-
-import java.io.Serializable;
-import java.util.function.Supplier;
-
-/**
- * Serializable supplier.
- *
- * @param <T> The type of results supplied by this supplier.
- */
-public interface SerializableSupplier<T> extends Supplier<T>, Serializable {
-}
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/package-info.java
deleted file mode 100644
index 408e16e..0000000
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Util classes used in {@link org.apache.ignite.tensorflow} package.
- */
-
-package org.apache.ignite.tensorflow.util;
diff --git a/modules/tensorflow/src/main/resources/logback.xml b/modules/tensorflow/src/main/resources/logback.xml
deleted file mode 100644
index 816b5e6..0000000
--- a/modules/tensorflow/src/main/resources/logback.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!--
- Logback configuration file.
--->
-<configuration>
-
- <appender name="FILE" class="ch.qos.logback.core.FileAppender">
- <file>ignite-tf.log</file>
- <append>false</append>
- <encoder>
- <pattern>%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
- </encoder>
- </appender>
-
- <root level="warn">
- <appender-ref ref="FILE" />
- </root>
-
-</configuration>
\ No newline at end of file
diff --git a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/TensorFlowTestSuite.java b/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/TensorFlowTestSuite.java
deleted file mode 100644
index 4ad5216..0000000
--- a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/TensorFlowTestSuite.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow;
-
-import org.apache.ignite.tensorflow.core.CoreTestSuite;
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-
-/**
- * Test suite for all module tests.
- */
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
- CoreTestSuite.class
-})
-public class TensorFlowTestSuite {
- // No-op.
-}
diff --git a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/CoreTestSuite.java b/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/CoreTestSuite.java
deleted file mode 100644
index 92b6dbd..0000000
--- a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/CoreTestSuite.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core;
-
-import org.apache.ignite.tensorflow.core.longrunning.LongRunningProcessManagerTest;
-import org.apache.ignite.tensorflow.core.longrunning.task.LongRunningProcessClearTaskTest;
-import org.apache.ignite.tensorflow.core.longrunning.task.LongRunningProcessPingTaskTest;
-import org.apache.ignite.tensorflow.core.longrunning.task.LongRunningProcessStartTaskTest;
-import org.apache.ignite.tensorflow.core.longrunning.task.LongRunningProcessStopTaskTest;
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-
-/**
- * Test suite for all tests in core package.
- */
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
- ProcessManagerWrapperTest.class,
- LongRunningProcessClearTaskTest.class,
- LongRunningProcessPingTaskTest.class,
- LongRunningProcessStartTaskTest.class,
- LongRunningProcessStopTaskTest.class,
- LongRunningProcessManagerTest.class
-})
-public class CoreTestSuite {
- // No-op.
-}
diff --git a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/ProcessManagerWrapperTest.java b/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/ProcessManagerWrapperTest.java
deleted file mode 100644
index e44daac..0000000
--- a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/ProcessManagerWrapperTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mock;
-import org.mockito.runners.MockitoJUnitRunner;
-
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.verify;
-
-/**
- * Tests for {@link ProcessManagerWrapper}.
- */
-@RunWith(MockitoJUnitRunner.class)
-public class ProcessManagerWrapperTest {
- /** Delegate. */
- @Mock
- private ProcessManager<String> delegate;
-
- /** Process manager wrapper. */
- private ProcessManagerWrapper<String, Integer> wrapper;
-
- /** Initializes tests. */
- @Before
- public void init() {
- wrapper = new TestProcessManagerWrapper(delegate);
- }
-
- /** */
- @Test
- public void testStart() {
- wrapper.start(Arrays.asList(1, 2, 3));
-
- verify(delegate).start(eq(Arrays.asList("1", "2", "3")));
- }
-
- /** */
- @Test
- public void testPing() {
- Map<UUID, List<UUID>> procIds = Collections.emptyMap();
- wrapper.ping(procIds);
-
- verify(delegate).ping(eq(procIds));
- }
-
- /** */
- @Test
- public void testStop() {
- Map<UUID, List<UUID>> procIds = Collections.emptyMap();
- wrapper.stop(procIds, true);
-
- verify(delegate).stop(eq(procIds), eq(true));
- }
-
- /** */
- @Test
- public void testClear() {
- Map<UUID, List<UUID>> procIds = Collections.emptyMap();
- wrapper.clear(procIds);
-
- verify(delegate).clear(eq(procIds));
- }
-
- /**
- * Process manager wrapper to be used in tests.
- */
- private static class TestProcessManagerWrapper extends ProcessManagerWrapper<String, Integer> {
- /** */
- private static final long serialVersionUID = 7562628311662129855L;
-
- /**
- * Constructs a new instance of process manager wrapper.
- *
- * @param delegate Delegate.
- */
- public TestProcessManagerWrapper(ProcessManager<String> delegate) {
- super(delegate);
- }
-
- /** {@inheritDoc} */
- @Override protected String transformSpecification(Integer spec) {
- return spec.toString();
- }
- }
-}
diff --git a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManagerTest.java b/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManagerTest.java
deleted file mode 100644
index faa2b6b..0000000
--- a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManagerTest.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCluster;
-import org.apache.ignite.IgniteCompute;
-import org.apache.ignite.cluster.ClusterGroup;
-import org.apache.ignite.lang.IgniteCallable;
-import org.apache.ignite.tensorflow.core.longrunning.task.LongRunningProcessClearTask;
-import org.apache.ignite.tensorflow.core.longrunning.task.LongRunningProcessPingTask;
-import org.apache.ignite.tensorflow.core.longrunning.task.LongRunningProcessStartTask;
-import org.apache.ignite.tensorflow.core.longrunning.task.LongRunningProcessStopTask;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessState;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-
-/**
- * Tests for {@link LongRunningProcessManager}.
- */
-public class LongRunningProcessManagerTest {
- /** */
- @Test
- @SuppressWarnings("unchecked")
- public void testStart() {
- UUID nodeId = UUID.randomUUID();
- UUID procId = UUID.randomUUID();
-
- Ignite ignite = mock(Ignite.class);
- IgniteCluster cluster = mock(IgniteCluster.class);
- ClusterGroup clusterGrp = mock(ClusterGroup.class);
- IgniteCompute igniteCompute = mock(IgniteCompute.class);
- doReturn(cluster).when(ignite).cluster();
- doReturn(igniteCompute).when(ignite).compute(eq(clusterGrp));
- doReturn(clusterGrp).when(cluster).forNodeId(eq(nodeId));
- doReturn(Collections.singletonList(procId)).when(igniteCompute).call(any(IgniteCallable.class));
-
- List<LongRunningProcess> list = Collections.singletonList(new LongRunningProcess(nodeId, () -> {}));
-
- LongRunningProcessManager mgr = new LongRunningProcessManager(ignite);
- Map<UUID, List<UUID>> res = mgr.start(list);
-
- assertEquals(1, res.size());
- assertTrue(res.containsKey(nodeId));
- assertEquals(procId, res.get(nodeId).iterator().next());
-
- verify(igniteCompute).call(any(LongRunningProcessStartTask.class));
- }
-
- /** */
- @Test
- @SuppressWarnings("unchecked")
- public void testPing() {
- UUID nodeId = UUID.randomUUID();
- UUID procId = UUID.randomUUID();
-
- Ignite ignite = mock(Ignite.class);
- IgniteCluster cluster = mock(IgniteCluster.class);
- ClusterGroup clusterGrp = mock(ClusterGroup.class);
- IgniteCompute igniteCompute = mock(IgniteCompute.class);
- doReturn(cluster).when(ignite).cluster();
- doReturn(igniteCompute).when(ignite).compute(eq(clusterGrp));
- doReturn(clusterGrp).when(cluster).forNodeId(eq(nodeId));
- doReturn(Collections.singletonList(new LongRunningProcessStatus(LongRunningProcessState.RUNNING)))
- .when(igniteCompute).call(any(IgniteCallable.class));
-
- Map<UUID, List<UUID>> procIds = new HashMap<>();
- procIds.put(nodeId, Collections.singletonList(procId));
-
- LongRunningProcessManager mgr = new LongRunningProcessManager(ignite);
- Map<UUID, List<LongRunningProcessStatus>> res = mgr.ping(procIds);
-
- assertEquals(1, res.size());
- assertTrue(res.containsKey(nodeId));
- assertEquals(LongRunningProcessState.RUNNING, res.get(nodeId).iterator().next().getState());
-
- verify(igniteCompute).call(any(LongRunningProcessPingTask.class));
- }
-
- /** */
- @Test
- @SuppressWarnings("unchecked")
- public void testStop() {
- UUID nodeId = UUID.randomUUID();
- UUID procId = UUID.randomUUID();
-
- Ignite ignite = mock(Ignite.class);
- IgniteCluster cluster = mock(IgniteCluster.class);
- ClusterGroup clusterGrp = mock(ClusterGroup.class);
- IgniteCompute igniteCompute = mock(IgniteCompute.class);
- doReturn(cluster).when(ignite).cluster();
- doReturn(igniteCompute).when(ignite).compute(eq(clusterGrp));
- doReturn(clusterGrp).when(cluster).forNodeId(eq(nodeId));
- doReturn(Collections.singletonList(new LongRunningProcessStatus(LongRunningProcessState.RUNNING)))
- .when(igniteCompute).call(any(IgniteCallable.class));
-
- Map<UUID, List<UUID>> procIds = new HashMap<>();
- procIds.put(nodeId, Collections.singletonList(procId));
-
- LongRunningProcessManager mgr = new LongRunningProcessManager(ignite);
- Map<UUID, List<LongRunningProcessStatus>> res = mgr.stop(procIds, true);
-
- assertEquals(1, res.size());
- assertTrue(res.containsKey(nodeId));
- assertEquals(LongRunningProcessState.RUNNING, res.get(nodeId).iterator().next().getState());
-
- verify(igniteCompute).call(any(LongRunningProcessStopTask.class));
- }
-
- /** */
- @Test
- @SuppressWarnings("unchecked")
- public void testClear() {
- UUID nodeId = UUID.randomUUID();
- UUID procId = UUID.randomUUID();
-
- Ignite ignite = mock(Ignite.class);
- IgniteCluster cluster = mock(IgniteCluster.class);
- ClusterGroup clusterGrp = mock(ClusterGroup.class);
- IgniteCompute igniteCompute = mock(IgniteCompute.class);
- doReturn(cluster).when(ignite).cluster();
- doReturn(igniteCompute).when(ignite).compute(eq(clusterGrp));
- doReturn(clusterGrp).when(cluster).forNodeId(eq(nodeId));
- doReturn(Collections.singletonList(new LongRunningProcessStatus(LongRunningProcessState.RUNNING)))
- .when(igniteCompute).call(any(IgniteCallable.class));
-
- Map<UUID, List<UUID>> procIds = new HashMap<>();
- procIds.put(nodeId, Collections.singletonList(procId));
-
- LongRunningProcessManager mgr = new LongRunningProcessManager(ignite);
- Map<UUID, List<LongRunningProcessStatus>> res = mgr.clear(procIds);
-
- assertEquals(1, res.size());
- assertTrue(res.containsKey(nodeId));
- assertEquals(LongRunningProcessState.RUNNING, res.get(nodeId).iterator().next().getState());
-
- verify(igniteCompute).call(any(LongRunningProcessClearTask.class));
- }
-}
diff --git a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessClearTaskTest.java b/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessClearTaskTest.java
deleted file mode 100644
index 268f3e4..0000000
--- a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessClearTaskTest.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning.task;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessState;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-
-/**
- * Tests for {@link LongRunningProcessClearTask}.
- */
-public class LongRunningProcessClearTaskTest {
- /** Process metadata storage used instead of Apache Ignite node local storage. */
- private final ConcurrentMap<UUID, Future<?>> metadataStorage = new ConcurrentHashMap<>();
-
- /** Initializes tests. */
- @Before
- public void init() {
- metadataStorage.clear();
- }
-
- /** Tests execution of the task in case process is not found. */
- @Test
- public void testCallProcessNotFound() {
- LongRunningProcessClearTask clearTask = createTask(UUID.randomUUID());
-
- List<LongRunningProcessStatus> statuses = clearTask.call();
-
- assertEquals(1, statuses.size());
-
- LongRunningProcessStatus status = statuses.get(0);
- assertEquals(LongRunningProcessState.NOT_FOUND, status.getState());
- assertNull(status.getException());
-
- assertEquals(0, metadataStorage.size());
- }
-
- /** Tests execution of the task in case process is running. */
- @Test(expected = IllegalStateException.class)
- public void testCallProcessIsRunning() {
- UUID procId = UUID.randomUUID();
-
- Future<?> fut = mock(Future.class);
- doReturn(false).when(fut).isDone();
- metadataStorage.put(procId, fut);
-
- LongRunningProcessClearTask clearTask = createTask(procId);
-
- clearTask.call();
- }
-
- /** Tests execution of the task in case process is done. */
- @Test
- public void testCallProcessIsDone() {
- UUID procId = UUID.randomUUID();
-
- Future<?> fut = mock(Future.class);
- doReturn(true).when(fut).isDone();
- metadataStorage.put(procId, fut);
-
- LongRunningProcessClearTask clearTask = createTask(procId);
-
- List<LongRunningProcessStatus> statuses = clearTask.call();
-
- assertEquals(1, statuses.size());
-
- LongRunningProcessStatus status = statuses.get(0);
- assertEquals(LongRunningProcessState.DONE, status.getState());
- assertNull(status.getException());
-
- assertEquals(0, metadataStorage.size());
- }
-
- /** Tests execution of the task in case process is done with exception. */
- @Test
- public void testCallProcessIsDoneWithException() throws ExecutionException, InterruptedException {
- UUID procId = UUID.randomUUID();
-
- Future<?> fut = mock(Future.class);
- doReturn(true).when(fut).isDone();
- doThrow(RuntimeException.class).when(fut).get();
- metadataStorage.put(procId, fut);
-
- LongRunningProcessClearTask clearTask = createTask(procId);
-
- List<LongRunningProcessStatus> statuses = clearTask.call();
-
- assertEquals(1, statuses.size());
-
- LongRunningProcessStatus status = statuses.get(0);
- assertEquals(LongRunningProcessState.DONE, status.getState());
- assertNotNull(status.getException());
- assertTrue(status.getException() instanceof RuntimeException);
-
- assertEquals(0, metadataStorage.size());
- }
-
- /**
- * Creates clear task.
- *
- * @param procId Process identifier.
- * @return Clear task.
- */
- private LongRunningProcessClearTask createTask(UUID procId) {
- LongRunningProcessClearTask clearTask = new LongRunningProcessClearTask(Collections.singletonList(procId));
-
- clearTask = spy(clearTask);
- doReturn(metadataStorage).when(clearTask).getMetadataStorage();
-
- return clearTask;
- }
-}
diff --git a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessPingTaskTest.java b/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessPingTaskTest.java
deleted file mode 100644
index 818509a..0000000
--- a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessPingTaskTest.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning.task;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessState;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-
-/**
- * Tests for {@link LongRunningProcessPingTask}.
- */
-public class LongRunningProcessPingTaskTest {
- /** Process metadata storage used instead of Apache Ignite node local storage. */
- private final ConcurrentMap<UUID, Future<?>> metadataStorage = new ConcurrentHashMap<>();
-
- /** Initializes tests. */
- @Before
- public void init() {
- metadataStorage.clear();
- }
-
- /** Tests execution of the task in case process is not found. */
- @Test
- public void testCallProcessNotFound() {
- LongRunningProcessPingTask pingTask = createTask(UUID.randomUUID());
-
- List<LongRunningProcessStatus> statuses = pingTask.call();
-
- assertEquals(1, statuses.size());
-
- LongRunningProcessStatus status = statuses.get(0);
- assertEquals(LongRunningProcessState.NOT_FOUND, status.getState());
- assertNull(status.getException());
-
- assertEquals(0, metadataStorage.size());
- }
-
- /** Tests execution of the task in case process is running. */
- @Test
- public void testCallProcessIsRunning() {
- UUID procId = UUID.randomUUID();
-
- Future<?> fut = mock(Future.class);
- doReturn(false).when(fut).isDone();
- metadataStorage.put(procId, fut);
-
- LongRunningProcessPingTask pingTask = createTask(procId);
-
- List<LongRunningProcessStatus> statuses = pingTask.call();
-
- assertEquals(1, statuses.size());
-
- LongRunningProcessStatus status = statuses.get(0);
- assertEquals(LongRunningProcessState.RUNNING, status.getState());
- assertNull(status.getException());
-
- assertEquals(1, metadataStorage.size());
- }
-
- /** Tests execution of the task in case process is done. */
- @Test
- public void testCallProcessIsDone() {
- UUID procId = UUID.randomUUID();
-
- Future<?> fut = mock(Future.class);
- doReturn(true).when(fut).isDone();
- metadataStorage.put(procId, fut);
-
- LongRunningProcessPingTask pingTask = createTask(procId);
-
- List<LongRunningProcessStatus> statuses = pingTask.call();
-
- assertEquals(1, statuses.size());
-
- LongRunningProcessStatus status = statuses.get(0);
- assertEquals(LongRunningProcessState.DONE, status.getState());
- assertNull(status.getException());
-
- assertEquals(1, metadataStorage.size());
- }
-
- /** Tests execution of the task in case process is done with exception. */
- @Test
- public void testCallProcessIsDoneWithException() throws ExecutionException, InterruptedException {
- UUID procId = UUID.randomUUID();
-
- Future<?> fut = mock(Future.class);
- doReturn(true).when(fut).isDone();
- doThrow(RuntimeException.class).when(fut).get();
- metadataStorage.put(procId, fut);
-
- LongRunningProcessPingTask pingTask = createTask(procId);
-
- List<LongRunningProcessStatus> statuses = pingTask.call();
-
- assertEquals(1, statuses.size());
-
- LongRunningProcessStatus status = statuses.get(0);
- assertEquals(LongRunningProcessState.DONE, status.getState());
- assertNotNull(status.getException());
- assertTrue(status.getException() instanceof RuntimeException);
-
- assertEquals(1, metadataStorage.size());
- }
-
- /**
- * Creates ping task.
- *
- * @param procId Process identifier.
- * @return Ping task.
- */
- private LongRunningProcessPingTask createTask(UUID procId) {
- LongRunningProcessPingTask clearTask = new LongRunningProcessPingTask(Collections.singletonList(procId));
-
- clearTask = spy(clearTask);
- doReturn(metadataStorage).when(clearTask).getMetadataStorage();
-
- return clearTask;
- }
-}
diff --git a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStartTaskTest.java b/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStartTaskTest.java
deleted file mode 100644
index 0d14335..0000000
--- a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStartTaskTest.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning.task;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import org.apache.ignite.tensorflow.core.longrunning.LongRunningProcess;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.spy;
-
-/**
- * Tests for {@link LongRunningProcessStartTask}.
- */
-public class LongRunningProcessStartTaskTest {
- /** Process metadata storage used instead of Apache Ignite node local storage. */
- private final ConcurrentMap<UUID, Future<?>> metadataStorage = new ConcurrentHashMap<>();
-
- /** Initializes tests. */
- @Before
- public void init() {
- metadataStorage.clear();
- }
-
- /** */
- @Test
- public void testCall() throws ExecutionException, InterruptedException {
- LongRunningProcess proc = new LongRunningProcess(UUID.randomUUID(), () -> {});
- LongRunningProcessStartTask task = createTask(proc);
- List<UUID> procIds = task.call();
-
- assertEquals(1, procIds.size());
-
- UUID procId = procIds.get(0);
-
- assertNotNull(metadataStorage.get(procId));
-
- Future<?> fut = metadataStorage.get(procId);
- fut.get();
-
- assertEquals(true, fut.isDone());
- }
-
- /** */
- @Test(expected = ExecutionException.class)
- public void testCallWithException() throws ExecutionException, InterruptedException {
- LongRunningProcess proc = new LongRunningProcess(UUID.randomUUID(), () -> {
- throw new RuntimeException();
- });
- LongRunningProcessStartTask task = createTask(proc);
- List<UUID> procIds = task.call();
-
- assertEquals(1, procIds.size());
-
- UUID procId = procIds.get(0);
-
- assertNotNull(metadataStorage.get(procId));
-
- Future<?> fut = metadataStorage.get(procId);
- fut.get();
- }
-
- /**
- * Creates start task.
- *
- * @param proc Long running process.
- * @return Start task.
- */
- private LongRunningProcessStartTask createTask(LongRunningProcess proc) {
- LongRunningProcessStartTask startTask = new LongRunningProcessStartTask(Collections.singletonList(proc));
-
- startTask = spy(startTask);
- doReturn(metadataStorage).when(startTask).getMetadataStorage();
-
- return startTask;
- }
-}
diff --git a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStopTaskTest.java b/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStopTaskTest.java
deleted file mode 100644
index 129f0cb..0000000
--- a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStopTaskTest.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.core.longrunning.task;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessState;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-
-/**
- * Tests for {@link LongRunningProcessStopTask}.
- */
-public class LongRunningProcessStopTaskTest {
- /** Process metadata storage used instead of Apache Ignite node local storage. */
- private final ConcurrentMap<UUID, Future<?>> metadataStorage = new ConcurrentHashMap<>();
-
- /** Initializes tests. */
- @Before
- public void init() {
- metadataStorage.clear();
- }
-
- /** Tests execution of the task in case process is not found. */
- @Test
- public void testCallProcessNotFound() {
- LongRunningProcessStopTask stopTask = createTask(UUID.randomUUID(), true);
-
- List<LongRunningProcessStatus> statuses = stopTask.call();
-
- assertEquals(1, statuses.size());
-
- LongRunningProcessStatus status = statuses.get(0);
- assertEquals(LongRunningProcessState.NOT_FOUND, status.getState());
- assertNull(status.getException());
-
- assertEquals(0, metadataStorage.size());
- }
-
- /** Tests execution of the task in case process is running. */
- @Test
- public void testCallProcessIsRunning() {
- UUID procId = UUID.randomUUID();
-
- Future<?> fut = mock(Future.class);
- doReturn(false).when(fut).isDone();
- metadataStorage.put(procId, fut);
-
- LongRunningProcessStopTask stopTask = createTask(procId, true);
-
- List<LongRunningProcessStatus> statuses = stopTask.call();
-
- assertEquals(1, statuses.size());
- verify(fut).cancel(eq(true));
-
- LongRunningProcessStatus status = statuses.get(0);
- assertEquals(LongRunningProcessState.DONE, status.getState());
- assertNull(status.getException());
-
- assertEquals(0, metadataStorage.size());
- }
-
- /** Tests execution of the task in case process is done. */
- @Test
- public void testCallProcessIsDone() {
- UUID procId = UUID.randomUUID();
-
- Future<?> fut = mock(Future.class);
- doReturn(true).when(fut).isDone();
- metadataStorage.put(procId, fut);
-
- LongRunningProcessStopTask stopTask = createTask(procId, true);
-
- List<LongRunningProcessStatus> statuses = stopTask.call();
-
- assertEquals(1, statuses.size());
- verify(fut).cancel(eq(true));
-
- LongRunningProcessStatus status = statuses.get(0);
- assertEquals(LongRunningProcessState.DONE, status.getState());
- assertNull(status.getException());
-
- assertEquals(0, metadataStorage.size());
- }
-
- /** Tests execution of the task in case process is done with exception. */
- @Test
- public void testCallProcessIsDoneWithException() throws ExecutionException, InterruptedException {
- UUID procId = UUID.randomUUID();
-
- Future<?> fut = mock(Future.class);
- doReturn(true).when(fut).isDone();
- doThrow(RuntimeException.class).when(fut).get();
- metadataStorage.put(procId, fut);
-
- LongRunningProcessStopTask stopTask = createTask(procId, true);
-
- List<LongRunningProcessStatus> statuses = stopTask.call();
-
- assertEquals(1, statuses.size());
- verify(fut).cancel(eq(true));
-
- LongRunningProcessStatus status = statuses.get(0);
- assertEquals(LongRunningProcessState.DONE, status.getState());
- assertNotNull(status.getException());
- assertTrue(status.getException() instanceof RuntimeException);
-
- assertEquals(0, metadataStorage.size());
- }
-
- /**
- * Creates stop task.
- *
- * @param procId Process identifier.
- * @return Stop task.
- */
- private LongRunningProcessStopTask createTask(UUID procId, boolean clear) {
- LongRunningProcessStopTask clearTask = new LongRunningProcessStopTask(Collections.singletonList(procId), clear);
-
- clearTask = Mockito.spy(clearTask);
- doReturn(metadataStorage).when(clearTask).getMetadataStorage();
-
- return clearTask;
- }
-}
diff --git a/parent/pom.xml b/parent/pom.xml
index 8b92c5b..f2b883c 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -465,10 +465,6 @@
<packages>org.apache.ignite.ml*</packages>
</group>
<group>
- <title>Ignite TensorFlow</title>
- <packages>org.apache.ignite.tensorflow*</packages>
- </group>
- <group>
<title>Encryption SPI</title>
<packages>org.apache.ignite.spi.encryption*</packages>
</group>
diff --git a/pom.xml b/pom.xml
index 6db5b55..28e0385 100644
--- a/pom.xml
+++ b/pom.xml
@@ -96,9 +96,7 @@
<module>modules/ml</module>
<module>modules/ml/spark-model-parser</module>
<module>modules/ml/xgboost-model-parser</module>
- <module>modules/ml/tensorflow-model-parser</module>
<module>modules/ml/h2o-model-parser</module>
- <module>modules/tensorflow</module>
<module>modules/opencensus</module>
</modules>
@@ -122,7 +120,6 @@
<profile>
<id>all-scala</id><!-- used to update project versions and check all modules compilation -->
<modules><!-- sorted alphabetically -->
- <module>modules/ml/mleap-model-parser</module>
<module>modules/scalar-2.10</module>
<module>modules/scalar</module>
<module>modules/spark</module>
@@ -560,7 +557,6 @@
</activation>
<modules>
- <module>modules/ml/mleap-model-parser</module>
<module>modules/scalar</module>
<module>modules/spark</module>
<module>modules/visor-console</module>
@@ -577,7 +573,6 @@
<modules>
<module>modules/spark-2.4</module>
- <module>modules/ml/mleap-model-parser</module>
<module>modules/scalar</module>
</modules>
</profile>
