HORN-7: Implementation of DistBelief package for model parallelism
diff --git a/CHANGES.txt b/CHANGES.txt
new file mode 100644
index 0000000..0826b90
--- /dev/null
+++ b/CHANGES.txt
@@ -0,0 +1,11 @@
+Horn Change Log
+
+Release 0.1.0 (unreleased changes)
+
+ NEW FEATURES
+
+ HORN-1: Website for Apache Horn (Incubating) (Elmurod Talipov via edwardyoon)
+
+ BUG FIXES
+
+ IMPROVEMENTS
diff --git a/src/main/java/org/apache/horn/distbelief/Neuron.java b/src/main/java/org/apache/horn/distbelief/Neuron.java
new file mode 100644
index 0000000..ce67cf2
--- /dev/null
+++ b/src/main/java/org/apache/horn/distbelief/Neuron.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.horn.distbelief;
+
+import org.apache.hadoop.io.Writable;
+
+public abstract class Neuron<M extends Writable> implements NeuronInterface<M> {
+ double output;
+ double weight;
+
+ public void setOutput(double output) {
+ this.output = output;
+ }
+
+ public double getOutput() {
+ return output;
+ }
+
+ public void push(double weight) {
+ this.weight = weight;
+ }
+
+ public double getUpdate() {
+ return weight;
+ }
+
+}
diff --git a/src/main/java/org/apache/horn/distbelief/NeuronInterface.java b/src/main/java/org/apache/horn/distbelief/NeuronInterface.java
new file mode 100644
index 0000000..8093b07
--- /dev/null
+++ b/src/main/java/org/apache/horn/distbelief/NeuronInterface.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.horn.distbelief;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+
+public interface NeuronInterface<M extends Writable> {
+
+ /**
+ * This method is called when the messages are propagated from the lower
+ * layer. It can be used to determine if the neuron would activate, or fire.
+ *
+ * @param messages
+ * @throws IOException
+ */
+ public void upward(Iterable<M> messages) throws IOException;
+
+ /**
+ * This method is called when the errors are propagated from the upper layer.
+ * It can be used to calculate the error of each neuron and change the
+ * weights.
+ *
+ * @param messages
+ * @throws IOException
+ */
+ public void downward(Iterable<M> messages) throws IOException;
+
+}
diff --git a/src/main/java/org/apache/horn/distbelief/PropMessage.java b/src/main/java/org/apache/horn/distbelief/PropMessage.java
new file mode 100644
index 0000000..dd6f2b1
--- /dev/null
+++ b/src/main/java/org/apache/horn/distbelief/PropMessage.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.horn.distbelief;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Message wrapper for a propagating message
+ */
+public class PropMessage<M extends Writable, W extends Writable> implements
+ Writable {
+
+ M message;
+ W weight;
+
+ public PropMessage(M message, W weight) {
+ this.message = message;
+ this.weight = weight;
+ }
+
+ public M getMessage() {
+ return message;
+ }
+
+ public W getWeight() {
+ return weight;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ message.readFields(in);
+ weight.readFields(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ message.write(out);
+ weight.write(out);
+ }
+
+}
diff --git a/src/test/java/org/apache/horn/distbelief/TestNeuron.java b/src/test/java/org/apache/horn/distbelief/TestNeuron.java
new file mode 100644
index 0000000..37e8fd6
--- /dev/null
+++ b/src/test/java/org/apache/horn/distbelief/TestNeuron.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.horn.distbelief;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hama.commons.math.Sigmoid;
+
+public class TestNeuron extends TestCase {
+ private static double learningRate = 0.1;
+
+ public static class MyNeuron extends
+ Neuron<PropMessage<DoubleWritable, DoubleWritable>> {
+
+ @Override
+ public void upward(
+ Iterable<PropMessage<DoubleWritable, DoubleWritable>> messages)
+ throws IOException {
+ double sum = 0;
+ for (PropMessage<DoubleWritable, DoubleWritable> m : messages) {
+ sum += m.getMessage().get() * m.getWeight().get();
+ }
+ sum += (-1 * 0.8);
+
+ double output = new Sigmoid().apply(sum);
+ this.setOutput(output);
+ }
+
+ @Override
+ public void downward(
+ Iterable<PropMessage<DoubleWritable, DoubleWritable>> messages)
+ throws IOException {
+
+ for (PropMessage<DoubleWritable, DoubleWritable> m : messages) {
+ // Calculates error gradient for each neuron
+ double gradient = this.getOutput() * (1 - this.getOutput())
+ * m.getMessage().get() * m.getWeight().get();
+
+ // Propagates to lower layer
+ System.out.println(gradient);
+
+ // Weight corrections
+ double weight = learningRate * this.getOutput() * m.getMessage().get();
+ this.push(weight);
+ }
+ }
+
+ }
+
+ public void testProp() throws IOException {
+ List<PropMessage<DoubleWritable, DoubleWritable>> x = new ArrayList<PropMessage<DoubleWritable, DoubleWritable>>();
+ x.add(new PropMessage<DoubleWritable, DoubleWritable>(new DoubleWritable(
+ 1.0), new DoubleWritable(0.5)));
+ x.add(new PropMessage<DoubleWritable, DoubleWritable>(new DoubleWritable(
+ 1.0), new DoubleWritable(0.4)));
+
+ MyNeuron n = new MyNeuron();
+ n.upward(x);
+ assertEquals(0.5249791874789399, n.getOutput());
+
+ x.clear();
+ x.add(new PropMessage<DoubleWritable, DoubleWritable>(new DoubleWritable(
+ -0.1274), new DoubleWritable(-1.2)));
+ n.downward(x);
+ assertEquals(-0.006688234848481696, n.getUpdate());
+ }
+}
