HORN-21: Add dropout neuron
diff --git a/README.md b/README.md
index 4c9ec6d..bbad6ab 100644
--- a/README.md
+++ b/README.md
@@ -23,10 +23,10 @@
public void backward(
Iterable<Synapse<FloatWritable, FloatWritable>> messages)
throws IOException {
- float gradient = 0;
+ float delta = 0;
for (Synapse<FloatWritable, FloatWritable> m : messages) {
// Calculates error gradient for each neuron
- gradient += (m.getDelta() * m.getWeight());
+ delta += (m.getDelta() * m.getWeight());
// Weight corrections
float weight = -this.getLearningRate() * this.getOutput()
@@ -34,23 +34,22 @@
this.push(weight);
}
- this.backpropagate(gradient
+ this.backpropagate(delta
* this.squashingFunction.applyDerivative(this.getOutput()));
}
}
```
The advantages of this programming model is easy and intuitive to use.
-Also, Apache Horn provides a simplified and intuitive configuration interface. To create neural network job and submit it to existing Hadoop or Hama cluster, we just add the layer with its properties such as squashing function and neuron class. The below example configures the create 2-layer neural network with 100 neurons in hidden layers for train MNIST dataset:
+Also, Apache Horn provides a simplified and intuitive configuration interface. To create neural network job and submit it to existing Hadoop or Hama cluster, we just add the layer with its properties such as squashing function and neuron class. The below example configures the create 2-layer neural network for train MNIST dataset:
```Java
HornJob job = new HornJob(conf, MultiLayerPerceptron.class);
job.setLearningRate(learningRate);
- job.setTrainingMethod(TrainingMethod.GRADIENT_DESCENT);
..
- job.inputLayer(784, Sigmoid.class, StandardNeuron.class);
- job.addLayer(100, Sigmoid.class, StandardNeuron.class);
- job.outputLayer(10, SoftMax.class, StandardNeuron.class);
+ job.inputLayer(features, 0.8f); // droprate
+ job.addLayer(hu, ReLU.class, DropoutNeuron.class);
+ job.outputLayer(labels, SoftMax.class, StandardNeuron.class);
job.setCostFunction(CrossEntropy.class);
```
@@ -68,7 +67,7 @@
0.01 0.9 0.0005 784 100 10 10 12000
```
-With this default example, you'll reach over the 95% accuracy. In local mode, 6 tasks will train the model in synchronous parallel fashion and will took around 30 mins.
+With this default example, you'll reach over the 95% accuracy. In local mode, 20 tasks will train the model in synchronous parallel fashion and will took around 10 mins.
## High Scalability
diff --git a/conf/log4j.properties b/conf/log4j.properties
index 8e3b877..f33e9a8 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -67,7 +67,8 @@
log4j.logger.org.apache.zookeeper=ERROR
log4j.logger.org.apache.avro=ERROR
-log4j.logger.org.apache.hama=INFO
+log4j.logger.org.apache.hama=ERROR
+log4j.logger.org.apache.hadoop=ERROR
#
# console
diff --git a/src/main/java/org/apache/horn/core/AbstractLayeredNeuralNetwork.java b/src/main/java/org/apache/horn/core/AbstractLayeredNeuralNetwork.java
index 4d1ea52..26e815f 100644
--- a/src/main/java/org/apache/horn/core/AbstractLayeredNeuralNetwork.java
+++ b/src/main/java/org/apache/horn/core/AbstractLayeredNeuralNetwork.java
@@ -155,7 +155,7 @@
* @return The layer index, starts with 0.
*/
public abstract int addLayer(int size, boolean isFinalLayer,
- FloatFunction squashingFunction, Class<? extends Neuron> neuronClass);
+ FloatFunction squashingFunction, Class<? extends Neuron<?>> neuronClass);
/**
* Get the size of a particular layer.
diff --git a/src/main/java/org/apache/horn/core/HornJob.java b/src/main/java/org/apache/horn/core/HornJob.java
index d178166..3912b67 100644
--- a/src/main/java/org/apache/horn/core/HornJob.java
+++ b/src/main/java/org/apache/horn/core/HornJob.java
@@ -35,36 +35,39 @@
super(conf);
this.setJarByClass(exampleClass);
+ // default local file block size 10mb
+ this.getConfiguration().set("fs.local.block.size", "10358951");
neuralNetwork = new LayeredNeuralNetwork();
}
- @SuppressWarnings("rawtypes")
- public void inputLayer(int featureDimension, Class<? extends Function> func,
- Class<? extends Neuron> neuronClass) {
- addLayer(featureDimension, func, neuronClass);
+ public void inputLayer(int featureDimension) {
+ addLayer(featureDimension, null, null);
+ neuralNetwork.setDropRateOfInputLayer(1);
}
- @SuppressWarnings("rawtypes")
+ public void inputLayer(int featureDimension, float dropRate) {
+ addLayer(featureDimension, null, null);
+ neuralNetwork.setDropRateOfInputLayer(dropRate);
+ }
+
public void addLayer(int featureDimension, Class<? extends Function> func,
- Class<? extends Neuron> neuronClass) {
- neuralNetwork
- .addLayer(featureDimension, false,
- FunctionFactory.createFloatFunction(func.getSimpleName()),
- neuronClass);
+ Class<? extends Neuron<?>> neuronClass) {
+ neuralNetwork.addLayer(
+ featureDimension,
+ false,
+ (func != null) ? FunctionFactory.createFloatFunction(func
+ .getSimpleName()) : null, neuronClass);
}
- @SuppressWarnings("rawtypes")
public void outputLayer(int labels, Class<? extends Function> func,
- Class<? extends Neuron> neuronClass) {
- neuralNetwork
- .addLayer(labels, true,
- FunctionFactory.createFloatFunction(func.getSimpleName()),
- neuronClass);
+ Class<? extends Neuron<?>> neuronClass) {
+ neuralNetwork.addLayer(labels, true,
+ FunctionFactory.createFloatFunction(func.getSimpleName()), neuronClass);
}
public void setCostFunction(Class<? extends Function> func) {
- neuralNetwork.setCostFunction(FunctionFactory
- .createFloatFloatFunction(func.getSimpleName()));
+ neuralNetwork.setCostFunction(FunctionFactory.createFloatFloatFunction(func
+ .getSimpleName()));
}
public void setDouble(String name, double value) {
@@ -82,15 +85,15 @@
public void setTrainingMethod(TrainingMethod method) {
this.neuralNetwork.setTrainingMethod(method);
}
-
+
public void setLearningStyle(LearningStyle style) {
this.neuralNetwork.setLearningStyle(style);
}
-
+
public void setLearningRate(float learningRate) {
this.neuralNetwork.setLearningRate(learningRate);
}
-
+
public void setConvergenceCheckInterval(int n) {
this.conf.setInt("convergence.check.interval", n);
}
@@ -98,7 +101,7 @@
public void setMomentumWeight(float momentumWeight) {
this.neuralNetwork.setMomemtumWeight(momentumWeight);
}
-
+
public void setRegularizationWeight(float regularizationWeight) {
this.neuralNetwork.setRegularizationWeight(regularizationWeight);
}
diff --git a/src/main/java/org/apache/horn/core/LayeredNeuralNetwork.java b/src/main/java/org/apache/horn/core/LayeredNeuralNetwork.java
index aa8e68d..6f7aa70 100644
--- a/src/main/java/org/apache/horn/core/LayeredNeuralNetwork.java
+++ b/src/main/java/org/apache/horn/core/LayeredNeuralNetwork.java
@@ -22,6 +22,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import org.apache.commons.lang.math.RandomUtils;
@@ -50,6 +51,7 @@
import org.apache.horn.funcs.FunctionFactory;
import org.apache.horn.funcs.IdentityFunction;
import org.apache.horn.funcs.SoftMax;
+import org.apache.horn.utils.MathUtils;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -79,11 +81,14 @@
/* Different layers can have different squashing function */
protected List<FloatFunction> squashingFunctionList;
- protected List<Class<? extends Neuron>> neuronClassList;
+ protected List<Class<? extends Neuron<?>>> neuronClassList;
protected int finalLayerIdx;
- private List<Neuron[]> neurons = new ArrayList<Neuron[]>();
+ private List<Neuron<?>[]> neurons = new ArrayList<Neuron<?>[]>();
+
+ private float dropRate;
+ private long iterations;
public LayeredNeuralNetwork() {
this.layerSizeList = Lists.newArrayList();
@@ -95,19 +100,28 @@
public LayeredNeuralNetwork(HamaConfiguration conf, String modelPath) {
super(conf, modelPath);
+ initializeNeurons(false);
+ }
- // initialize neuron objects
+ public LayeredNeuralNetwork(HamaConfiguration conf, String modelPath,
+ boolean isTraining) {
+ super(conf, modelPath);
+ initializeNeurons(isTraining);
+ }
+
+ // initialize neuron objects
+ private void initializeNeurons(boolean isTraining) {
for (int i = 0; i < layerSizeList.size(); i++) {
int numOfNeurons = layerSizeList.get(i);
- Class neuronClass;
+ Class<? extends Neuron<?>> neuronClass;
if (i == 0)
- neuronClass = Neuron.class;
+ neuronClass = StandardNeuron.class; // actually doesn't needed
else
neuronClass = neuronClassList.get(i - 1);
- Neuron[] tmp = new Neuron[numOfNeurons];
+ Neuron<?>[] tmp = new Neuron[numOfNeurons];
for (int j = 0; j < numOfNeurons; j++) {
- Neuron n = newNeuronInstance(StandardNeuron.class);
+ Neuron<?> n = newNeuronInstance(neuronClass);
if (i > 0)
n.setSquashingFunction(squashingFunctionList.get(i - 1));
else
@@ -115,8 +129,10 @@
n.setLayerIndex(i);
+ n.setNeuronID(j);
n.setLearningRate(this.learningRate);
n.setMomentumWeight(this.momentumWeight);
+ n.setTraining(isTraining);
tmp[j] = n;
}
@@ -129,12 +145,12 @@
* {@inheritDoc}
*/
public int addLayer(int size, boolean isFinalLayer,
- FloatFunction squashingFunction, Class<? extends Neuron> neuronClass) {
+ FloatFunction squashingFunction, Class<? extends Neuron<?>> neuronClass) {
return addLayer(size, isFinalLayer, squashingFunction, neuronClass, null);
}
public int addLayer(int size, boolean isFinalLayer,
- FloatFunction squashingFunction, Class<? extends Neuron> neuronClass,
+ FloatFunction squashingFunction, Class<? extends Neuron<?>> neuronClass,
Class<? extends IntermediateOutput> interlayer) {
Preconditions.checkArgument(size > 0,
"Size of layer must be larger than 0.");
@@ -267,13 +283,14 @@
super.readFields(input);
this.finalLayerIdx = input.readInt();
+ this.dropRate = input.readFloat();
// read neuron classes
int neuronClasses = input.readInt();
this.neuronClassList = Lists.newArrayList();
for (int i = 0; i < neuronClasses; ++i) {
try {
- Class<? extends Neuron> clazz = (Class<? extends Neuron>) Class
+ Class<? extends Neuron<?>> clazz = (Class<? extends Neuron<?>>) Class
.forName(input.readUTF());
neuronClassList.add(clazz);
} catch (ClassNotFoundException e) {
@@ -306,12 +323,12 @@
@Override
public void write(DataOutput output) throws IOException {
super.write(output);
-
output.writeInt(finalLayerIdx);
+ output.writeFloat(dropRate);
// write neuron classes
output.writeInt(this.neuronClassList.size());
- for (Class<? extends Neuron> clazz : this.neuronClassList) {
+ for (Class<? extends Neuron<?>> clazz : this.neuronClassList) {
output.writeUTF(clazz.getName());
}
@@ -361,6 +378,10 @@
return getOutputInternal(instanceWithBias);
}
+ public void setDropRateOfInputLayer(float dropRate) {
+ this.dropRate = dropRate;
+ }
+
/**
* Calculate output internally, the intermediate output of each layer will be
* stored.
@@ -370,9 +391,15 @@
*/
public FloatVector getOutputInternal(FloatVector instanceWithBias) {
// sets the output of input layer
- Neuron[] inputLayer = neurons.get(0);
+ Neuron<?>[] inputLayer = neurons.get(0);
for (int i = 0; i < inputLayer.length; i++) {
- inputLayer[i].setOutput(instanceWithBias.get(i));
+ float m2 = MathUtils.getBinomial(1, dropRate);
+ if(m2 == 0)
+ inputLayer[i].setDrop(true);
+ else
+ inputLayer[i].setDrop(false);
+
+ inputLayer[i].setOutput(instanceWithBias.get(i) * m2);
}
for (int i = 0; i < this.layerSizeList.size() - 1; ++i) {
@@ -397,6 +424,60 @@
return (Neuron) ReflectionUtils.newInstance(neuronClass);
}
+ public class InputMessageIterable implements
+ Iterable<Synapse<FloatWritable, FloatWritable>> {
+ private int currNeuronID;
+ private int prevNeuronID;
+ private int end;
+ private FloatMatrix weightMat;
+ private Neuron<?>[] layer;
+
+ public InputMessageIterable(int fromLayer, int row) {
+ this.currNeuronID = row;
+ this.prevNeuronID = -1;
+ this.end = weightMatrixList.get(fromLayer).getColumnCount() - 1;
+ this.weightMat = weightMatrixList.get(fromLayer);
+ this.layer = neurons.get(fromLayer);
+ }
+
+ @Override
+ public Iterator<Synapse<FloatWritable, FloatWritable>> iterator() {
+ return new MessageIterator();
+ }
+
+ private class MessageIterator implements
+ Iterator<Synapse<FloatWritable, FloatWritable>> {
+
+ @Override
+ public boolean hasNext() {
+ if (prevNeuronID < end) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private FloatWritable i = new FloatWritable();
+ private FloatWritable w = new FloatWritable();
+ private Synapse<FloatWritable, FloatWritable> msg = new Synapse<FloatWritable, FloatWritable>();
+
+ @Override
+ public Synapse<FloatWritable, FloatWritable> next() {
+ prevNeuronID++;
+
+ i.set(layer[prevNeuronID].getOutput());
+ w.set(weightMat.get(currNeuronID, prevNeuronID));
+ msg.set(prevNeuronID, i, w);
+ return new Synapse<FloatWritable, FloatWritable>(prevNeuronID, i, w);
+ }
+
+ @Override
+ public void remove() {
+ }
+
+ }
+ }
+
/**
* Forward the calculation for one layer.
*
@@ -410,20 +491,15 @@
FloatVector vec = new DenseFloatVector(weightMatrix.getRowCount());
for (int row = 0; row < weightMatrix.getRowCount(); row++) {
- List<Synapse<FloatWritable, FloatWritable>> msgs = new ArrayList<Synapse<FloatWritable, FloatWritable>>();
- for (int col = 0; col < weightMatrix.getColumnCount(); col++) {
- msgs.add(new Synapse<FloatWritable, FloatWritable>(new FloatWritable(
- neurons.get(fromLayer)[col].getOutput()), new FloatWritable(
- weightMatrix.get(row, col))));
- }
-
- Neuron n;
+ Neuron<?> n;
if (curLayerIdx == finalLayerIdx)
n = neurons.get(curLayerIdx)[row];
else
n = neurons.get(curLayerIdx)[row + 1];
try {
+ Iterable msgs = new InputMessageIterable(fromLayer, row);
+ n.setIterationNumber(iterations);
n.forward(msgs);
} catch (IOException e) {
// TODO Auto-generated catch block
@@ -518,7 +594,8 @@
calculateTrainingError(labels, output);
if (this.trainingMethod.equals(TrainingMethod.GRADIENT_DESCENT)) {
- return this.trainByInstanceGradientDescent(labels);
+ FloatMatrix[] updates = this.trainByInstanceGradientDescent(labels);
+ return updates;
} else {
throw new IllegalArgumentException(
String.format("Training method is not supported."));
@@ -563,7 +640,7 @@
costFuncDerivative *= squashingFunction.applyDerivative(finalOut);
}
- neurons.get(finalLayerIdx)[i].setDelta(costFuncDerivative);
+ neurons.get(finalLayerIdx)[i].backpropagate(costFuncDerivative);
deltaVec.set(i, costFuncDerivative);
}
@@ -576,6 +653,75 @@
return weightUpdateMatrices;
}
+ public class ErrorMessageIterable implements
+ Iterable<Synapse<FloatWritable, FloatWritable>> {
+ private int row;
+ private int neuronID;
+ private int end;
+ private FloatMatrix weightMat;
+ private FloatMatrix prevWeightMat;
+
+ private float[] nextLayerDelta;
+
+ public ErrorMessageIterable(int curLayerIdx, int row) {
+ this.row = row;
+ this.neuronID = -1;
+ this.weightMat = weightMatrixList.get(curLayerIdx);
+ this.end = weightMat.getRowCount() - 1;
+ this.prevWeightMat = prevWeightUpdatesList.get(curLayerIdx);
+
+ Neuron<?>[] nextLayer = neurons.get(curLayerIdx + 1);
+ nextLayerDelta = new float[weightMat.getRowCount()];
+
+ for(int i = 0; i <= end; ++i) {
+ if (curLayerIdx + 1 == finalLayerIdx) {
+ nextLayerDelta[i] = nextLayer[i].getDelta();
+ } else {
+ nextLayerDelta[i] = nextLayer[i + 1].getDelta();
+ }
+ }
+ }
+
+ @Override
+ public Iterator<Synapse<FloatWritable, FloatWritable>> iterator() {
+ return new MessageIterator();
+ }
+
+ private class MessageIterator implements
+ Iterator<Synapse<FloatWritable, FloatWritable>> {
+
+ @Override
+ public boolean hasNext() {
+ if (neuronID < end) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private FloatWritable d = new FloatWritable();
+ private FloatWritable w = new FloatWritable();
+ private FloatWritable p = new FloatWritable();
+ private Synapse<FloatWritable, FloatWritable> msg = new Synapse<FloatWritable, FloatWritable>();
+
+ @Override
+ public Synapse<FloatWritable, FloatWritable> next() {
+ neuronID++;
+
+ d.set(nextLayerDelta[neuronID]);
+ w.set(weightMat.get(neuronID, row));
+ p.set(prevWeightMat.get(neuronID, row));
+ msg.set(neuronID, d, w, p);
+ return msg;
+ }
+
+ @Override
+ public void remove() {
+ }
+
+ }
+ }
+
/**
* Back-propagate the errors to from next layer to current layer. The weight
* updated information will be stored in the weightUpdateMatrices, and the
@@ -588,31 +734,18 @@
DenseFloatMatrix weightUpdateMatrix) {
// get layer related information
- FloatMatrix weightMatrix = this.weightMatrixList.get(curLayerIdx);
- FloatMatrix prevWeightMatrix = this.prevWeightUpdatesList.get(curLayerIdx);
+ int x = this.weightMatrixList.get(curLayerIdx).getColumnCount();
+ int y = this.weightMatrixList.get(curLayerIdx).getRowCount();
- FloatVector deltaVector = new DenseFloatVector(
- weightMatrix.getColumnCount());
-
- for (int row = 0; row < weightMatrix.getColumnCount(); ++row) {
- Neuron n = neurons.get(curLayerIdx)[row];
- n.setWeightVector(weightMatrix.getRowCount());
-
- List<Synapse<FloatWritable, FloatWritable>> msgs = new ArrayList<Synapse<FloatWritable, FloatWritable>>();
-
- for (int col = 0; col < weightMatrix.getRowCount(); ++col) {
- float deltaOfNextLayer;
- if (curLayerIdx + 1 == this.finalLayerIdx)
- deltaOfNextLayer = neurons.get(curLayerIdx + 1)[col].getDelta();
- else
- deltaOfNextLayer = neurons.get(curLayerIdx + 1)[col + 1].getDelta();
-
- msgs.add(new Synapse<FloatWritable, FloatWritable>(new FloatWritable(
- deltaOfNextLayer), new FloatWritable(weightMatrix.get(col, row)),
- new FloatWritable(prevWeightMatrix.get(col, row))));
- }
+ FloatVector deltaVector = new DenseFloatVector(x);
+ Neuron<?>[] ns = neurons.get(curLayerIdx);
+
+ for (int row = 0; row < x; ++row) {
+ Neuron<?> n = ns[row];
+ n.setWeightVector(y);
try {
+ Iterable msgs = new ErrorMessageIterable(curLayerIdx, row);
n.backward(msgs);
} catch (IOException e) {
// TODO Auto-generated catch block
@@ -652,6 +785,7 @@
job.getConfiguration().setInt(Constants.ADDITIONAL_BSP_TASKS, 1);
+ job.setBoolean("training.mode", true);
job.setInputPath(new Path(conf.get("training.input.path")));
job.setInputFormat(org.apache.hama.bsp.SequenceFileInputFormat.class);
job.setInputKeyClass(LongWritable.class);
@@ -680,4 +814,8 @@
return this.squashingFunctionList.get(idx);
}
+ public void setIterationNumber(long iterations) {
+ this.iterations = iterations;
+ }
+
}
diff --git a/src/main/java/org/apache/horn/core/LayeredNeuralNetworkTrainer.java b/src/main/java/org/apache/horn/core/LayeredNeuralNetworkTrainer.java
index e0810e2..259c1a9 100644
--- a/src/main/java/org/apache/horn/core/LayeredNeuralNetworkTrainer.java
+++ b/src/main/java/org/apache/horn/core/LayeredNeuralNetworkTrainer.java
@@ -78,9 +78,9 @@
this.maxIterations = conf.getLong("training.max.iterations", Long.MAX_VALUE);
this.convergenceCheckInterval = conf.getLong("convergence.check.interval",
100);
- this.inMemoryModel = new LayeredNeuralNetwork(conf, modelPath);
+ this.inMemoryModel = new LayeredNeuralNetwork(conf, modelPath, true);
this.prevAvgTrainingError = Integer.MAX_VALUE;
- this.batchSize = conf.getInt("training.batch.size", 50);
+ this.batchSize = conf.getInt("training.batch.size", 5);
}
@Override
@@ -117,8 +117,14 @@
FloatVector v = value.getVector();
trainingSet.add(v);
}
-
+
+ if (peer.getPeerIndex() != peer.getNumPeers() - 1) {
+ LOG.debug(peer.getPeerName() + ": " + trainingSet.size() + " training instances loaded.");
+ }
+
while (this.iterations++ < maxIterations) {
+ this.inMemoryModel.setIterationNumber(iterations);
+
// each groom calculate the matrices updates according to local data
if (peer.getPeerIndex() != peer.getNumPeers() - 1) {
calculateUpdates(peer);
@@ -188,6 +194,7 @@
this.inMemoryModel.trainByInstance(trainingInstance));
avgTrainingError += this.inMemoryModel.trainingError;
}
+
avgTrainingError /= batchSize;
// calculate the average of updates
diff --git a/src/main/java/org/apache/horn/core/Neuron.java b/src/main/java/org/apache/horn/core/Neuron.java
index 908abf4..7519201 100644
--- a/src/main/java/org/apache/horn/core/Neuron.java
+++ b/src/main/java/org/apache/horn/core/Neuron.java
@@ -24,7 +24,8 @@
import org.apache.hadoop.io.Writable;
import org.apache.hama.commons.math.FloatFunction;
-public abstract class Neuron<M extends Writable> implements Writable, NeuronInterface<M> {
+public abstract class Neuron<M extends Writable> implements Writable,
+ NeuronInterface<M> {
int id;
float output;
float weight;
@@ -35,17 +36,20 @@
int layerIndex;
boolean isOutputLayer;
-
+ boolean isTraining;
+ boolean isDropped;
+ long iterations;
+
protected FloatFunction squashingFunction;
public void setNeuronID(int id) {
this.id = id;
}
-
- public int getID() {
+
+ public int getNeuronID() {
return id;
}
-
+
public int getLayerIndex() {
return layerIndex;
}
@@ -53,7 +57,7 @@
public void setLayerIndex(int index) {
this.layerIndex = index;
}
-
+
public void feedforward(float sum) {
this.output = sum;
}
@@ -62,10 +66,6 @@
this.delta = gradient;
}
- public void setDelta(float delta) {
- this.delta = delta;
- }
-
public float getDelta() {
return delta;
}
@@ -101,6 +101,7 @@
// ////////
private int i;
+ float[] weights;
public void push(float weight) {
weights[i++] = weight;
@@ -110,8 +111,6 @@
return weight;
}
- float[] weights;
-
public void setWeightVector(int rowCount) {
i = 0;
weights = new float[rowCount];
@@ -121,19 +120,34 @@
return weights;
}
+ public void setWeights(float[] weights) {
+ this.weights = weights;
+ }
+
public void setSquashingFunction(FloatFunction squashingFunction) {
this.squashingFunction = squashingFunction;
}
+ public void setTraining(boolean b) {
+ this.isTraining = b;
+ }
+
+ public boolean isTraining() {
+ return isTraining;
+ }
+
@Override
public void readFields(DataInput in) throws IOException {
id = in.readInt();
output = in.readFloat();
weight = in.readFloat();
delta = in.readFloat();
+ iterations = in.readLong();
momentumWeight = in.readFloat();
learningRate = in.readFloat();
+ isTraining = in.readBoolean();
+ isDropped = in.readBoolean();
}
@Override
@@ -142,9 +156,39 @@
out.writeFloat(output);
out.writeFloat(weight);
out.writeFloat(delta);
-
+ out.writeLong(iterations);
+
out.writeFloat(momentumWeight);
out.writeFloat(learningRate);
+ out.writeBoolean(isTraining);
+ out.writeBoolean(isDropped);
+ }
+
+ public void setIterationNumber(long iterations) {
+ this.iterations = iterations;
+ }
+
+ public long getIterationNumber() {
+ return iterations;
+ }
+
+ public boolean isDropped() {
+ return isDropped;
+ }
+
+ public void setDrop(boolean isDropped) {
+ this.isDropped = isDropped;
+ }
+
+ private float nablaW;
+
+ public void setNablaW(float f) {
+ // TODO Auto-generated method stub
+ nablaW = f;
+ }
+
+ public float getNablaW() {
+ return nablaW;
}
}
diff --git a/src/main/java/org/apache/horn/core/Synapse.java b/src/main/java/org/apache/horn/core/Synapse.java
index 7e9db2a..ae34400 100644
--- a/src/main/java/org/apache/horn/core/Synapse.java
+++ b/src/main/java/org/apache/horn/core/Synapse.java
@@ -30,21 +30,44 @@
public class Synapse<M extends Writable, W extends Writable> implements
Writable {
+ int neuronID;
FloatWritable message;
FloatWritable weight;
FloatWritable prevWeight;
- public Synapse(FloatWritable message, FloatWritable weight) {
+ public Synapse() {
+ }
+
+ public Synapse(int neuronID, FloatWritable message, FloatWritable weight) {
+ this.neuronID = neuronID;
this.message = message;
this.weight = weight;
}
- public Synapse(FloatWritable message, FloatWritable weight, FloatWritable prevWeight) {
+ public Synapse(int neuronID, FloatWritable message, FloatWritable weight, FloatWritable prevWeight) {
+ this.neuronID = neuronID;
this.message = message;
this.weight = weight;
this.prevWeight = prevWeight;
}
+ public void set(int neuronID, FloatWritable message, FloatWritable weight) {
+ this.neuronID = neuronID;
+ this.message = message;
+ this.weight = weight;
+ }
+
+ public void set(int neuronID, FloatWritable message, FloatWritable weight, FloatWritable prevWeight) {
+ this.neuronID = neuronID;
+ this.message = message;
+ this.weight = weight;
+ this.prevWeight = prevWeight;
+ }
+
+ public int getSenderID() {
+ return neuronID;
+ }
+
/**
* @return the activation or error message
*/
@@ -74,12 +97,16 @@
public void readFields(DataInput in) throws IOException {
message.readFields(in);
weight.readFields(in);
+ prevWeight.readFields(in);
+ neuronID = in.readInt();
}
@Override
public void write(DataOutput out) throws IOException {
message.write(out);
weight.write(out);
+ prevWeight.write(out);
+ out.writeInt(neuronID);
}
}
\ No newline at end of file
diff --git a/src/main/java/org/apache/horn/examples/DropoutNeuron.java b/src/main/java/org/apache/horn/examples/DropoutNeuron.java
new file mode 100644
index 0000000..ec02570
--- /dev/null
+++ b/src/main/java/org/apache/horn/examples/DropoutNeuron.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.horn.examples;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.horn.core.Neuron;
+import org.apache.horn.core.Synapse;
+import org.apache.horn.utils.MathUtils;
+
+public class DropoutNeuron extends
+ Neuron<Synapse<FloatWritable, FloatWritable>> {
+
+ private float m2;
+
+ @Override
+ public void forward(Iterable<Synapse<FloatWritable, FloatWritable>> messages)
+ throws IOException {
+ m2 = (isTraining()) ? MathUtils.getBinomial(1, 0.5) : 0.5f;
+
+ if (m2 > 0) {
+ float sum = 0;
+ for (Synapse<FloatWritable, FloatWritable> m : messages) {
+ sum += m.getInput() * m.getWeight();
+ }
+
+ this.setDrop(false);
+ this.feedforward(squashingFunction.apply(sum) * m2);
+ } else {
+ this.setDrop(true);
+ this.feedforward(0);
+ }
+ }
+
+ @Override
+ public void backward(Iterable<Synapse<FloatWritable, FloatWritable>> messages)
+ throws IOException {
+ if (!this.isDropped()) {
+ float delta = 0;
+
+ for (Synapse<FloatWritable, FloatWritable> m : messages) {
+ // Calculates error gradient for each neuron
+ delta += (m.getDelta() * m.getWeight());
+
+ // Weight corrections
+ float weight = -this.getLearningRate() * m.getDelta()
+ * this.getOutput() + this.getMomentumWeight() * m.getPrevWeight();
+ this.push(weight);
+ }
+
+ this.backpropagate(delta * squashingFunction.applyDerivative(getOutput()));
+ } else {
+ this.backpropagate(0);
+ }
+ }
+
+}
diff --git a/src/main/java/org/apache/horn/examples/MultiLayerPerceptron.java b/src/main/java/org/apache/horn/examples/MultiLayerPerceptron.java
index a787dda..5f3403b 100644
--- a/src/main/java/org/apache/horn/examples/MultiLayerPerceptron.java
+++ b/src/main/java/org/apache/horn/examples/MultiLayerPerceptron.java
@@ -26,7 +26,7 @@
import org.apache.horn.core.Neuron;
import org.apache.horn.core.Synapse;
import org.apache.horn.funcs.CrossEntropy;
-import org.apache.horn.funcs.Sigmoid;
+import org.apache.horn.funcs.ReLU;
import org.apache.horn.funcs.SoftMax;
public class MultiLayerPerceptron {
@@ -35,8 +35,7 @@
Neuron<Synapse<FloatWritable, FloatWritable>> {
@Override
- public void forward(
- Iterable<Synapse<FloatWritable, FloatWritable>> messages)
+ public void forward(Iterable<Synapse<FloatWritable, FloatWritable>> messages)
throws IOException {
float sum = 0;
for (Synapse<FloatWritable, FloatWritable> m : messages) {
@@ -49,19 +48,21 @@
public void backward(
Iterable<Synapse<FloatWritable, FloatWritable>> messages)
throws IOException {
- float gradient = 0;
- for (Synapse<FloatWritable, FloatWritable> m : messages) {
- // Calculates error gradient for each neuron
- gradient += (m.getDelta() * m.getWeight());
+ float delta = 0;
- // Weight corrections
- float weight = -this.getLearningRate() * this.getOutput()
- * m.getDelta() + this.getMomentumWeight() * m.getPrevWeight();
- this.push(weight);
+ if (!this.isDropped()) {
+ for (Synapse<FloatWritable, FloatWritable> m : messages) {
+ // Calculates error gradient for each neuron
+ delta += (m.getDelta() * m.getWeight());
+
+ // Weight corrections
+ float weight = -this.getLearningRate() * m.getDelta()
+ * this.getOutput() + this.getMomentumWeight() * m.getPrevWeight();
+ this.push(weight);
+ }
}
- this.backpropagate(gradient
- * squashingFunction.applyDerivative(getOutput()));
+ this.backpropagate(delta * squashingFunction.applyDerivative(getOutput()));
}
}
@@ -79,13 +80,13 @@
job.setMomentumWeight(momemtumWeight);
job.setRegularizationWeight(regularizationWeight);
- job.setConvergenceCheckInterval(1000);
+ job.setConvergenceCheckInterval(100);
job.setBatchSize(miniBatch);
job.setTrainingMethod(TrainingMethod.GRADIENT_DESCENT);
- job.inputLayer(features, Sigmoid.class, StandardNeuron.class);
- job.addLayer(hu, Sigmoid.class, StandardNeuron.class);
+ job.inputLayer(features, 0.8f); // droprate
+ job.addLayer(hu, ReLU.class, DropoutNeuron.class);
job.outputLayer(labels, SoftMax.class, StandardNeuron.class);
job.setCostFunction(CrossEntropy.class);
@@ -111,7 +112,7 @@
long startTime = System.currentTimeMillis();
if (ann.waitForCompletion(true)) {
- System.out.println("Job Finished in "
+ System.out.println("Optimization Finished! "
+ (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");
}
}
diff --git a/src/main/java/org/apache/horn/funcs/CategoricalCrossEntropy.java b/src/main/java/org/apache/horn/funcs/CategoricalCrossEntropy.java
index 887f24d..3e9bfac 100644
--- a/src/main/java/org/apache/horn/funcs/CategoricalCrossEntropy.java
+++ b/src/main/java/org/apache/horn/funcs/CategoricalCrossEntropy.java
@@ -20,15 +20,15 @@
import org.apache.hama.commons.math.FloatFloatFunction;
/**
- * for softmaxed output
+ * for softmaxed output
*/
public class CategoricalCrossEntropy extends FloatFloatFunction {
-
- private static final float epsilon = (float) 1e-8;
-
+
+ private static final float epsilon = 1e-8f;
+
@Override
public float apply(float target, float actual) {
- return -target * (float) Math.log(Math.max(actual, epsilon));
+ return -target * (float) Math.log(Math.min(Math.max(actual, epsilon), 1.0f));
}
@Override
diff --git a/src/main/java/org/apache/horn/funcs/ReLU.java b/src/main/java/org/apache/horn/funcs/ReLU.java
index 2f14f54..a185d39 100644
--- a/src/main/java/org/apache/horn/funcs/ReLU.java
+++ b/src/main/java/org/apache/horn/funcs/ReLU.java
@@ -30,15 +30,15 @@
@Override
public float apply(float value) {
- return Math.max(0.001f, value);
+ return Math.max(0.0f, value);
}
@Override
public float applyDerivative(float value) {
if (value > 0)
- return 0.999f;
+ return 1.0f;
else
- return 0.001f;
+ return 0.0f;
}
}
diff --git a/src/main/java/org/apache/horn/funcs/SoftMax.java b/src/main/java/org/apache/horn/funcs/SoftMax.java
index 710b489..4e42202 100644
--- a/src/main/java/org/apache/horn/funcs/SoftMax.java
+++ b/src/main/java/org/apache/horn/funcs/SoftMax.java
@@ -20,7 +20,6 @@
import java.io.IOException;
import org.apache.hama.commons.math.DenseFloatVector;
-import org.apache.hama.commons.math.DoubleVector;
import org.apache.hama.commons.math.FloatFunction;
import org.apache.hama.commons.math.FloatVector;
import org.apache.horn.core.IntermediateOutput;
@@ -44,11 +43,13 @@
public FloatVector interlayer(FloatVector output) throws IOException {
FloatVector expVec = new DenseFloatVector(output.getDimension());
float sum = 0.0f;
+ float max = output.max(); // to avoid infinity
for(int i = 0; i < output.getDimension(); ++i) {
- float exp = (float) Math.exp(output.get(i));
+ float exp = (float) Math.exp(output.get(i) - max);
sum += exp;
expVec.set(i, exp);
}
+
// divide by the sum of exponential of the whole vector
FloatVector softmaxed = expVec.divide(sum);
return softmaxed;
diff --git a/src/main/java/org/apache/horn/utils/MNISTConverter.java b/src/main/java/org/apache/horn/utils/MNISTConverter.java
index 25ea2a0..e13b9ff 100644
--- a/src/main/java/org/apache/horn/utils/MNISTConverter.java
+++ b/src/main/java/org/apache/horn/utils/MNISTConverter.java
@@ -49,6 +49,10 @@
String labels_data = args[1];
String output = args[2];
+ HamaConfiguration conf = new HamaConfiguration();
+ conf.set("dfs.block.size", "11554432");
+ FileSystem fs = FileSystem.get(conf);
+
DataInputStream imagesIn = new DataInputStream(new FileInputStream(
new File(training_data)));
DataInputStream labelsIn = new DataInputStream(new FileInputStream(
@@ -70,9 +74,6 @@
labels[n] = labelsIn.readByte();
}
- HamaConfiguration conf = new HamaConfiguration();
- FileSystem fs = FileSystem.get(conf);
-
@SuppressWarnings("deprecation")
SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, new Path(
output), LongWritable.class, FloatVectorWritable.class);
diff --git a/src/main/java/org/apache/horn/utils/MNISTEvaluator.java b/src/main/java/org/apache/horn/utils/MNISTEvaluator.java
index ede0d3e..a41fdb3 100644
--- a/src/main/java/org/apache/horn/utils/MNISTEvaluator.java
+++ b/src/main/java/org/apache/horn/utils/MNISTEvaluator.java
@@ -21,7 +21,6 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
-import java.util.Random;
import org.apache.hama.HamaConfiguration;
import org.apache.hama.commons.math.DenseFloatVector;
@@ -60,41 +59,30 @@
imagesIn.readInt(); // Rows
imagesIn.readInt(); // Cols
- byte[][] images = new byte[count][PIXELS];
- byte[] labels = new byte[count];
- for (int n = 0; n < count; n++) {
- imagesIn.readFully(images[n]);
- labels[n] = labelsIn.readByte();
- }
-
HamaConfiguration conf = new HamaConfiguration();
LayeredNeuralNetwork ann = new LayeredNeuralNetwork(conf, modelPath);
- Random generator = new Random();
int correct = 0;
int total = 0;
- for (int i = 0; i < count; i++) {
- if (generator.nextInt(10) == 1) {
- float[] vals = new float[PIXELS];
- for (int j = 0; j < PIXELS; j++) {
- vals[j] = rescale((images[i][j] & 0xff));
- }
- int label = (labels[i] & 0xff);
- FloatVector instance = new DenseFloatVector(vals);
- FloatVector result = ann.getOutput(instance);
+ for (int n = 0; n < count; n++) {
+ byte[] vector = new byte[PIXELS];
+ imagesIn.readFully(vector);
+ int label = (labelsIn.readByte() & 0xff);
- if (getNumber(result) == label) {
- correct++;
- }
- total++;
+ FloatVector instance = new DenseFloatVector(PIXELS);
+ for (int j = 0; j < PIXELS; j++) {
+ instance.set(j, rescale((vector[j] & 0xff)));
}
+ FloatVector result = ann.getOutput(instance);
+
+ if (getNumber(result) == label) {
+ correct++;
+ }
+ total++;
}
- System.out.println(((double) correct / total * 100) + "%");
- // TODO System.out.println("Precision = " + (tp / (tp + fp)));
- // System.out.println("Recall = " + (tp / (tp + fn)));
- // System.out.println("Accuracy = " + ((tp + tn) / (tp + tn + fp + fn)));
+ System.out.println("Accuracy: " + ((double) correct / total));
imagesIn.close();
labelsIn.close();
diff --git a/src/main/java/org/apache/horn/utils/MathUtils.java b/src/main/java/org/apache/horn/utils/MathUtils.java
new file mode 100644
index 0000000..164e221
--- /dev/null
+++ b/src/main/java/org/apache/horn/utils/MathUtils.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.horn.utils;
+
+public class MathUtils {
+
+ public static int getBinomial(int n, double p) {
+ int x = 0;
+ for (int i = 0; i < n; i++) {
+ if (Math.random() < p)
+ x++;
+ }
+ return x;
+ }
+
+}
diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties
index f7bed47..f2a431d 100644
--- a/src/main/resources/log4j.properties
+++ b/src/main/resources/log4j.properties
@@ -81,5 +81,5 @@
#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
#log4j.logger.org.apache.hadoop.dfs=DEBUG
-#log4j.logger.org.apache.hama=DEBUG
+log4j.logger.org.apache.hama=ERROR
#log4j.logger.org.apache.zookeeper=DEBUG
diff --git a/src/test/java/org/apache/horn/core/TestNeuron.java b/src/test/java/org/apache/horn/core/TestNeuron.java
index c962746..a05e531 100644
--- a/src/test/java/org/apache/horn/core/TestNeuron.java
+++ b/src/test/java/org/apache/horn/core/TestNeuron.java
@@ -71,9 +71,9 @@
public void testProp() throws IOException {
List<Synapse<FloatWritable, FloatWritable>> x = new ArrayList<Synapse<FloatWritable, FloatWritable>>();
- x.add(new Synapse<FloatWritable, FloatWritable>(new FloatWritable(1.0f),
+ x.add(new Synapse<FloatWritable, FloatWritable>(0, new FloatWritable(1.0f),
new FloatWritable(0.5f)));
- x.add(new Synapse<FloatWritable, FloatWritable>(new FloatWritable(1.0f),
+ x.add(new Synapse<FloatWritable, FloatWritable>(0, new FloatWritable(1.0f),
new FloatWritable(0.4f)));
MyNeuron n = new MyNeuron();
@@ -81,7 +81,7 @@
assertEquals(0.5249792f, n.getOutput());
x.clear();
- x.add(new Synapse<FloatWritable, FloatWritable>(new FloatWritable(
+ x.add(new Synapse<FloatWritable, FloatWritable>(0, new FloatWritable(
-0.1274f), new FloatWritable(-1.2f)));
n.backward(x);
}
diff --git a/src/test/java/org/apache/horn/examples/MultiLayerPerceptronTest.java b/src/test/java/org/apache/horn/examples/MultiLayerPerceptronTest.java
index 2e87659..74ccbb6 100644
--- a/src/test/java/org/apache/horn/examples/MultiLayerPerceptronTest.java
+++ b/src/test/java/org/apache/horn/examples/MultiLayerPerceptronTest.java
@@ -171,7 +171,7 @@
job.setTrainingSetPath(SEQTRAIN_DATA);
job.setModelPath(MODEL_PATH);
- job.setMaxIteration(1000);
+ job.setMaxIteration(1);
job.setLearningRate(0.4f);
job.setMomentumWeight(0.2f);
job.setRegularizationWeight(0.001f);
@@ -181,7 +181,7 @@
job.setTrainingMethod(TrainingMethod.GRADIENT_DESCENT);
- job.inputLayer(featureDimension, Sigmoid.class, StandardNeuron.class);
+ job.inputLayer(featureDimension, 0.8f);
job.addLayer(featureDimension, Sigmoid.class, StandardNeuron.class);
job.outputLayer(labelDimension, Sigmoid.class, StandardNeuron.class);
diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties
index f7bed47..6bcc0a4 100644
--- a/src/test/resources/log4j.properties
+++ b/src/test/resources/log4j.properties
@@ -81,5 +81,5 @@
#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
#log4j.logger.org.apache.hadoop.dfs=DEBUG
-#log4j.logger.org.apache.hama=DEBUG
+#log4j.logger.org.apache.hama=ERROR
#log4j.logger.org.apache.zookeeper=DEBUG
