Use hama-0.7.2-SNAPSHOT
diff --git a/pom.xml b/pom.xml
index 8e258d4..5f83793 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,8 +32,8 @@
<properties>
<commons-logging.version>1.1.1</commons-logging.version>
- <hama.version>0.7.0</hama.version>
- <hadoop.version>2.7.0</hadoop.version>
+ <hama.version>0.7.2-SNAPSHOT</hama.version>
+ <hadoop.version>2.7.2</hadoop.version>
<protobuf.version>2.5.0</protobuf.version>
<junit.version>4.8.1</junit.version>
<log4j.version>1.2.16</log4j.version>
diff --git a/src/main/java/org/apache/horn/bsp/AbstractLayeredNeuralNetwork.java b/src/main/java/org/apache/horn/bsp/AbstractLayeredNeuralNetwork.java
index 1afe8f5..b18eb44 100644
--- a/src/main/java/org/apache/horn/bsp/AbstractLayeredNeuralNetwork.java
+++ b/src/main/java/org/apache/horn/bsp/AbstractLayeredNeuralNetwork.java
@@ -28,7 +28,7 @@
import org.apache.hama.commons.math.DoubleFunction;
import org.apache.hama.commons.math.DoubleMatrix;
import org.apache.hama.commons.math.DoubleVector;
-import org.apache.hama.commons.math.FunctionFactory;
+import org.apache.horn.funcs.FunctionFactory;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
diff --git a/src/main/java/org/apache/horn/bsp/AutoEncoder.java b/src/main/java/org/apache/horn/bsp/AutoEncoder.java
index a632942..8ea2930 100644
--- a/src/main/java/org/apache/horn/bsp/AutoEncoder.java
+++ b/src/main/java/org/apache/horn/bsp/AutoEncoder.java
@@ -27,8 +27,8 @@
import org.apache.hama.commons.math.DoubleFunction;
import org.apache.hama.commons.math.DoubleMatrix;
import org.apache.hama.commons.math.DoubleVector;
-import org.apache.hama.commons.math.FunctionFactory;
import org.apache.hama.ml.util.FeatureTransformer;
+import org.apache.horn.funcs.FunctionFactory;
import com.google.common.base.Preconditions;
diff --git a/src/main/java/org/apache/horn/bsp/HornJob.java b/src/main/java/org/apache/horn/bsp/HornJob.java
index a9c7cc1..4521b87 100644
--- a/src/main/java/org/apache/horn/bsp/HornJob.java
+++ b/src/main/java/org/apache/horn/bsp/HornJob.java
@@ -22,7 +22,7 @@
import org.apache.hama.HamaConfiguration;
import org.apache.hama.bsp.BSPJob;
import org.apache.hama.commons.math.Function;
-import org.apache.hama.commons.math.FunctionFactory;
+import org.apache.horn.funcs.FunctionFactory;
public class HornJob extends BSPJob {
diff --git a/src/main/java/org/apache/horn/bsp/NeuralNetworkTrainer.java b/src/main/java/org/apache/horn/bsp/NeuralNetworkTrainer.java
index 1c8a198..648e86b 100644
--- a/src/main/java/org/apache/horn/bsp/NeuralNetworkTrainer.java
+++ b/src/main/java/org/apache/horn/bsp/NeuralNetworkTrainer.java
@@ -28,7 +28,6 @@
import org.apache.hama.bsp.BSPPeer;
import org.apache.hama.bsp.sync.SyncException;
import org.apache.hama.commons.io.VectorWritable;
-import org.apache.hama.ml.perception.MLPMessage;
import org.apache.hama.ml.util.DefaultFeatureTransformer;
import org.apache.hama.ml.util.FeatureTransformer;
@@ -39,7 +38,7 @@
*
*/
public abstract class NeuralNetworkTrainer extends
- BSP<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage> {
+ BSP<LongWritable, VectorWritable, NullWritable, NullWritable, Synapse> {
protected static final Log LOG = LogFactory
.getLog(NeuralNetworkTrainer.class);
@@ -53,7 +52,7 @@
@Override
final public void setup(
- BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage> peer)
+ BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, Synapse> peer)
throws IOException, SyncException, InterruptedException {
conf = peer.getConfiguration();
featureTransformer = new DefaultFeatureTransformer();
@@ -69,7 +68,7 @@
* @throws InterruptedException
*/
protected void extraSetup(
- BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage> peer)
+ BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, Synapse> peer)
throws IOException, SyncException, InterruptedException {
}
@@ -79,12 +78,12 @@
*/
@Override
public abstract void bsp(
- BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage> peer)
+ BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, Synapse> peer)
throws IOException, SyncException, InterruptedException;
@Override
public void cleanup(
- BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage> peer)
+ BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, Synapse> peer)
throws IOException {
this.extraCleanup(peer);
// write model to modelPath
@@ -99,7 +98,7 @@
* @throws InterruptedException
*/
protected void extraCleanup(
- BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage> peer)
+ BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, Synapse> peer)
throws IOException {
}
diff --git a/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetwork.java b/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetwork.java
index aaad86e..0ea8e51 100644
--- a/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetwork.java
+++ b/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetwork.java
@@ -33,6 +33,7 @@
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hama.Constants;
import org.apache.hama.HamaConfiguration;
import org.apache.hama.bsp.BSPJob;
import org.apache.hama.commons.io.MatrixWritable;
@@ -42,9 +43,9 @@
import org.apache.hama.commons.math.DoubleFunction;
import org.apache.hama.commons.math.DoubleMatrix;
import org.apache.hama.commons.math.DoubleVector;
-import org.apache.hama.commons.math.FunctionFactory;
import org.apache.hama.util.ReflectionUtils;
import org.apache.horn.examples.MultiLayerPerceptron.StandardNeuron;
+import org.apache.horn.funcs.FunctionFactory;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -583,6 +584,11 @@
job.getConfiguration().setClass("neuron.class", StandardNeuron.class,
Neuron.class);
+ // additional for parameter server
+ // TODO at this moment, we use 1 task as a parameter server
+ // In the future, the number of parameter server should be configurable
+ job.getConfiguration().setInt(Constants.ADDITIONAL_BSP_TASKS, 1);
+
job.setInputPath(new Path(conf.get("training.input.path")));
job.setInputFormat(org.apache.hama.bsp.SequenceFileInputFormat.class);
job.setInputKeyClass(LongWritable.class);
diff --git a/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetworkTrainer.java b/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetworkTrainer.java
index 58f96d1..c3e258c 100644
--- a/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetworkTrainer.java
+++ b/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetworkTrainer.java
@@ -33,7 +33,6 @@
import org.apache.hama.commons.math.DenseDoubleMatrix;
import org.apache.hama.commons.math.DoubleMatrix;
import org.apache.hama.commons.math.DoubleVector;
-import org.apache.hama.commons.math.FunctionFactory;
import org.apache.hama.ipc.RPC;
import com.google.common.base.Preconditions;
diff --git a/src/main/java/org/apache/horn/examples/NeuralNetwork.java b/src/main/java/org/apache/horn/examples/NeuralNetwork.java
index 1503ef0..5c0afdf 100644
--- a/src/main/java/org/apache/horn/examples/NeuralNetwork.java
+++ b/src/main/java/org/apache/horn/examples/NeuralNetwork.java
@@ -30,8 +30,8 @@
import org.apache.hama.HamaConfiguration;
import org.apache.hama.commons.math.DenseDoubleVector;
import org.apache.hama.commons.math.DoubleVector;
-import org.apache.hama.commons.math.FunctionFactory;
import org.apache.horn.bsp.SmallLayeredNeuralNetwork;
+import org.apache.horn.funcs.FunctionFactory;
/**
* The example of using {@link SmallLayeredNeuralNetwork}, including the
diff --git a/src/test/java/org/apache/horn/bsp/TestSmallLayeredNeuralNetwork.java b/src/test/java/org/apache/horn/bsp/TestSmallLayeredNeuralNetwork.java
index 972d55a..ee48136 100644
--- a/src/test/java/org/apache/horn/bsp/TestSmallLayeredNeuralNetwork.java
+++ b/src/test/java/org/apache/horn/bsp/TestSmallLayeredNeuralNetwork.java
@@ -44,11 +44,11 @@
import org.apache.hama.commons.math.DenseDoubleVector;
import org.apache.hama.commons.math.DoubleMatrix;
import org.apache.hama.commons.math.DoubleVector;
-import org.apache.hama.commons.math.FunctionFactory;
import org.apache.hama.ml.util.DefaultFeatureTransformer;
import org.apache.hama.ml.util.FeatureTransformer;
import org.apache.horn.bsp.AbstractLayeredNeuralNetwork.LearningStyle;
import org.apache.horn.bsp.AbstractLayeredNeuralNetwork.TrainingMethod;
+import org.apache.horn.funcs.FunctionFactory;
import org.junit.Test;
import org.mortbay.log.Log;
@@ -616,7 +616,6 @@
trainingParameters.put("training.batch.size", "300");
trainingParameters.put("convergence.check.interval", "1000");
//ann.train(new HamaConfiguration(), tmpDatasetPath, trainingParameters);
-
long end = new Date().getTime();
diff --git a/src/test/java/org/apache/horn/examples/NeuralNetworkTest.java b/src/test/java/org/apache/horn/examples/NeuralNetworkTest.java
index 5582dc0..932b17a 100644
--- a/src/test/java/org/apache/horn/examples/NeuralNetworkTest.java
+++ b/src/test/java/org/apache/horn/examples/NeuralNetworkTest.java
@@ -60,7 +60,6 @@
conf.get("bsp.master.address"));
conf.set("bsp.local.dir", "/tmp/hama-test");
conf.set(Constants.ZOOKEEPER_QUORUM, "localhost");
- conf.setBoolean(Constants.FORCE_SET_BSP_TASKS, true);
conf.setInt(Constants.ZOOKEEPER_CLIENT_PORT, 21810);
conf.set("hama.sync.client.class",
org.apache.hama.bsp.sync.ZooKeeperSyncClientImpl.class
