OPENNLP-757 Added some headers and fixed some issues. Thanks to Mondher Bouazizi for providing a patch.
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/Constants.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/Constants.java
index de8f9a4..9909c70 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/Constants.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/Constants.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package opennlp.tools.disambiguator;
import java.util.ArrayList;
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/DataExtractor.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/DataExtractor.java
new file mode 100644
index 0000000..0161504
--- /dev/null
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/DataExtractor.java
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package opennlp.tools.disambiguator;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import opennlp.tools.disambiguator.DictionaryInstance;
+import opennlp.tools.disambiguator.DistributionInstance;
+import opennlp.tools.disambiguator.ims.WTDIMS;
+
+public class DataExtractor {
+
+ /**
+ * Constructor
+ */
+
+ public DataExtractor() {
+ super();
+ }
+
+ /**
+ * Extract the dictionary from the dictionary XML file and map the senses
+ */
+
+ private ArrayList<DictionaryInstance> extractDictionary(String xmlLocation) {
+
+ ArrayList<DictionaryInstance> dictionary = new ArrayList<DictionaryInstance>();
+
+ // HashMap<Integer, DictionaryInstance> dictionary = new HashMap<Integer,
+ // DictionaryInstance>();
+
+ try {
+
+ File xmlFile = new File(xmlLocation);
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+ Document doc = dBuilder.parse(xmlFile);
+ doc.getDocumentElement().normalize();
+
+ NodeList nLexelts = doc.getElementsByTagName("lexelt");
+
+ int index = 0;
+
+ for (int i = 0; i < nLexelts.getLength(); i++) {
+
+ Node nLexelt = nLexelts.item(i);
+
+ Element eLexelt = (Element) nLexelt;
+
+ String word = eLexelt.getAttribute("item");
+
+ if (nLexelt.getNodeType() == Node.ELEMENT_NODE) {
+
+ NodeList nSenses = eLexelt.getChildNodes();
+
+ for (int j = 0; j < nSenses.getLength(); j++) {
+
+ if (nSenses.item(j).getNodeType() == Node.ELEMENT_NODE) {
+
+ Element eSense = (Element) nSenses.item(j);
+
+ int ind = index; // rather use this than the ID used by default
+ String id = eSense.getAttribute("id");
+ String source = eSense.getAttribute("source");
+ String[] synset = eSense.getAttribute("synset").split("\\s");
+ String gloss = eSense.getAttribute("gloss");
+
+ DictionaryInstance wd = new DictionaryInstance(ind, word, id,
+ source, synset, gloss);
+
+ dictionary.add(wd);
+ index++;
+ }
+ }
+
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return dictionary;
+
+ }
+
+ private HashMap<Integer, ArrayList<String>> getEquivalentSense(
+ String sensemapFile) {
+
+ HashMap<Integer, ArrayList<String>> mappedSenses = new HashMap<Integer, ArrayList<String>>();
+
+ try (BufferedReader wordsList = new BufferedReader(new FileReader(
+ sensemapFile))) {
+
+ int index = 0;
+
+ String line;
+
+ // Read the file
+ while ((line = wordsList.readLine()) != null) {
+
+ String[] temp = line.split("\\s");
+
+ ArrayList<String> tempSenses = new ArrayList<String>();
+
+ for (String sense : temp) {
+ if (sense.length() > 1) {
+ // System.out.println(sense);
+ tempSenses.add(sense);
+ }
+ }
+
+ mappedSenses.put(index, tempSenses);
+ // System.out.println(index);
+ index++;
+
+ }
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return mappedSenses;
+
+ }
+
+ private HashMap<String, ArrayList<DictionaryInstance>> extractOptimalDictionary(
+ String xmlLocation, String sensemapFile) {
+
+ HashMap<String, ArrayList<DictionaryInstance>> optimizedDictionary = new HashMap<String, ArrayList<DictionaryInstance>>();
+
+ HashMap<Integer, ArrayList<String>> equivalentSenses = getEquivalentSense(sensemapFile);
+
+ ArrayList<DictionaryInstance> dictionary = extractDictionary(xmlLocation);
+
+ for (int mapKey : equivalentSenses.keySet()) {
+ ArrayList<String> sensesIds = equivalentSenses.get(mapKey);
+ ArrayList<DictionaryInstance> optimizedDictionaryInstance = new ArrayList<DictionaryInstance>();
+
+ String word = "";
+
+ for (String senseId : sensesIds) {
+ for (int i = 0; i < dictionary.size(); i++) {
+ if (dictionary.get(i).getId().equals(senseId)) {
+ optimizedDictionaryInstance.add(dictionary.get(i));
+ word = dictionary.get(i).getWord();
+ word = word + "_" + mapKey;
+ break;
+ }
+ }
+
+ }
+
+ optimizedDictionary.put(word, optimizedDictionaryInstance);
+ }
+
+ return optimizedDictionary;
+ }
+
+ public HashMap<String, ArrayList<DictionaryInstance>> extractWordSenses(
+ String xmlLocation, String sensemapFile, String wordTag) {
+
+ /**
+ * word tag has to be in the format "word.t" (e.g., "activate.v", "smart.a",
+ * etc.)
+ */
+
+ HashMap<String, ArrayList<DictionaryInstance>> wordSenses = new HashMap<String, ArrayList<DictionaryInstance>>();
+
+ HashMap<String, ArrayList<DictionaryInstance>> optimalDictionary = extractOptimalDictionary(
+ xmlLocation, sensemapFile);
+
+ int i = 0;
+ for (String key : optimalDictionary.keySet()) {
+ if (key.startsWith(wordTag)) {
+ String newKey = wordTag + "_" + i;
+ wordSenses.put(newKey, optimalDictionary.get(key));
+ i++;
+ }
+ }
+
+ return wordSenses;
+ }
+
+ public HashMap<String, String> getDictionaryInstance(String xmlLocation,
+ String sensemapFile, String wordTag) {
+
+ HashMap<String, ArrayList<DictionaryInstance>> dict = extractWordSenses(
+ xmlLocation, sensemapFile, wordTag);
+
+ HashMap<String, String> senses = new HashMap<String, String>();
+
+ for (String key : dict.keySet()) {
+ String sense = dict.get(key).get(0).getGloss();
+ senses.put(key, sense);
+ }
+
+ return senses;
+
+ }
+
+ /**
+ * Extract the Dictionary Map [USELESS UNLESS USED FOR STATISTICS LATER !!!]
+ */
+
+ public HashMap<Integer, DistributionInstance> extractWords(String listOfWords) {
+
+ HashMap<Integer, DistributionInstance> instances = new HashMap<Integer, DistributionInstance>();
+
+ try (BufferedReader wordsList = new BufferedReader(new FileReader(
+ listOfWords))) {
+
+ String line;
+
+ int index = 0;
+
+ // Read the file
+ while ((line = wordsList.readLine()) != null) {
+
+ String[] temp = line.split("\\t");
+
+ String[] wordPos = temp[0].split("\\.");
+
+ String tag;
+
+ if (wordPos[1].equals("n")) {
+ tag = "noun";
+ } else if (wordPos[1].equals("v")) {
+ tag = "verb";
+ } else if (wordPos[1].equals("a")) {
+ tag = "adjective";
+ } else {
+ tag = "adverb";
+ }
+
+ DistributionInstance word = new DistributionInstance(wordPos[0], tag,
+ Integer.parseInt(temp[1]), Integer.parseInt(temp[2]));
+
+ instances.put(index, word);
+
+ index++;
+
+ }
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return instances;
+ }
+
+ /**
+ * Extract the training instances from the training/test set File
+ */
+
+ public HashMap<Integer, WTDIMS> extractWSDInstances(String xmlDataSet) {
+
+ HashMap<Integer, WTDIMS> setInstances = new HashMap<Integer, WTDIMS>();
+
+ try {
+
+ File xmlFile = new File(xmlDataSet);
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+ Document doc = dBuilder.parse(xmlFile);
+
+ doc.getDocumentElement().normalize();
+
+ NodeList lexelts = doc.getElementsByTagName("lexelt");
+
+ int index = 0;
+
+ for (int i = 0; i < lexelts.getLength(); i++) {
+
+ Node nLexelt = lexelts.item(i);
+
+ if (nLexelt.getNodeType() == Node.ELEMENT_NODE) {
+ Element eLexelt = (Element) nLexelt;
+
+ NodeList nInstances = nLexelt.getChildNodes();
+
+ for (int j = 1; j < nInstances.getLength(); j++) {
+
+ Node nInstance = nInstances.item(j);
+
+ if (nInstance.getNodeType() == Node.ELEMENT_NODE) {
+
+ Element eInstance = (Element) nInstance;
+
+ String[] wordPos = eLexelt.getAttribute("item").split("\\.");
+ String word = wordPos[0]; // Word
+ String tag; // Part of Speech
+
+ if (wordPos[1].equals("n")) {
+ tag = "noun";
+ } else if (wordPos[1].equals("v")) {
+ tag = "verb";
+ } else if (wordPos[1].equals("a")) {
+ tag = "adjective";
+ } else {
+ tag = "adverb";
+ }
+
+ String id = eInstance.getAttribute("id");
+ String source = eInstance.getAttribute("docsrc");
+
+ ArrayList<String> answers = new ArrayList<String>();
+ String sentence = "";
+ String rawWord = "";
+
+ NodeList nChildren = nInstance.getChildNodes();
+
+ for (int k = 1; k < nChildren.getLength(); k++) {
+ Node nChild = nChildren.item(k);
+
+ if (nChild.getNodeName().equals("answer")) {
+ // String answer =
+ // nChild.getAttributes().item(0).getTextContent();
+ String senseid = nChild.getAttributes().item(1)
+ .getTextContent();
+
+ String temp = senseid;
+ // String[] temp = { answer, senseid };
+ answers.add(temp);
+ }
+
+ if (nChild.getNodeName().equals("context")) {
+ sentence = ((Element) nChild).getTextContent();
+
+ if (nChild.hasChildNodes()) {
+ // textbefore =
+ // nChild.getChildNodes().item(0).getTextContent();
+ rawWord = nChild.getChildNodes().item(1).getTextContent();
+ // textAfter =
+ // nChild.getChildNodes().item(2).getTextContent();
+ // System.out.println(rawWord);
+ }
+ }
+
+ }
+
+ WTDIMS wordToDisambiguate = new WTDIMS(word, answers, sentence,
+ rawWord);
+ setInstances.put(index, wordToDisambiguate);
+ index++;
+ // System.out.print(index + "\t");
+ // System.out.println(wordToDisambiguate.toString());
+ }
+
+ }
+
+ }
+
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return setInstances;
+
+ }
+
+}
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/DictionaryInstance.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/DictionaryInstance.java
new file mode 100644
index 0000000..e53ffa8
--- /dev/null
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/DictionaryInstance.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package opennlp.tools.disambiguator;
+
+public class DictionaryInstance {
+
+
+ protected int index;
+
+ protected String word;
+
+ protected String id;
+ protected String source;
+ protected String[] synset;
+ protected String gloss;
+
+
+ public DictionaryInstance(int index, String word, String id, String source,
+ String[] synset, String gloss) {
+ super();
+ this.index = index;
+ this.word = word;
+ this.id = id;
+ this.source = source;
+ this.synset = synset;
+ this.gloss = gloss;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ public String getWord() {
+ return word;
+ }
+
+ public void setWord(String word) {
+ this.word = word;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getSource() {
+ return source;
+ }
+
+ public void setSource(String source) {
+ this.source = source;
+ }
+
+ public String[] getSynset() {
+ return synset;
+ }
+
+ public void setSynset(String[] synset) {
+ this.synset = synset;
+ }
+
+ public String getGloss() {
+ return gloss;
+ }
+
+ public void setGloss(String gloss) {
+ this.gloss = gloss;
+ }
+}
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/DistributionInstance.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/DistributionInstance.java
new file mode 100644
index 0000000..40db063
--- /dev/null
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/DistributionInstance.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package opennlp.tools.disambiguator;
+
+public class DistributionInstance {
+
+ protected String word;
+ protected String pos;
+ protected int trainingSetInstances;
+ protected int testSetInstances;
+
+ public DistributionInstance(String word, String pos,
+ int trainingSetInstances, int testSetInstances) {
+ super();
+ this.word = word;
+ this.pos = pos;
+ this.trainingSetInstances = trainingSetInstances;
+ this.testSetInstances = testSetInstances;
+ }
+
+ public String getWord() {
+ return word;
+ }
+
+ public void setWord(String word) {
+ this.word = word;
+ }
+
+ public String getPos() {
+ return pos;
+ }
+
+ public void setPos(String pos) {
+ this.pos = pos;
+ }
+
+ public int getTrainingSetInstances() {
+ return trainingSetInstances;
+ }
+
+ public void setTrainingSetInstances(int trainingSetInstances) {
+ this.trainingSetInstances = trainingSetInstances;
+ }
+
+ public int getTestSetInstances() {
+ return testSetInstances;
+ }
+
+ public void setTestSetInstances(int testSetInstances) {
+ this.testSetInstances = testSetInstances;
+ }
+}
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/FeaturesExtractor.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/FeaturesExtractor.java
new file mode 100644
index 0000000..abcff6c
--- /dev/null
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/FeaturesExtractor.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package opennlp.tools.disambiguator;
+
+
+
+
+import java.util.ArrayList;
+
+import opennlp.tools.disambiguator.ims.WTDIMS;
+
+public class FeaturesExtractor {
+
+ public FeaturesExtractor() {
+ super();
+ }
+
+ /**
+ * @Algorithm: IMS (It Makes Sense)
+ *
+ * The following methods serve to extract the features for the
+ * algorithm IMS.
+ *
+ * Three families of features are to be extracted: - PoS of
+ * Surrounding Words: it requires one parameter: "Window size" -
+ * Surrounding Words: no parameters are required - Local
+ * Collocations: it requires one parameter: "the n-gram"
+ *
+ */
+
+ // private methods
+ private String[] extractPosOfSurroundingWords(String[] sentence,
+ int wordIndex, int windowSize) {
+
+ String[] taggedSentence = Loader.getTagger().tag(sentence);
+
+ String[] tags = new String[2 * windowSize + 1];
+
+ int j = 0;
+
+ for (int i = wordIndex - windowSize; i < wordIndex + windowSize; i++) {
+ if (i < 0 || i >= sentence.length) {
+ tags[j] = "null";
+ } else {
+ tags[j] = taggedSentence[i].toLowerCase();
+ }
+ j++;
+ }
+
+ return tags;
+ }
+
+ private String[] extractSurroundingWords(String[] sentence, int wordIndex) {
+
+ String[] posTags = Loader.getTagger().tag(sentence);
+
+ ArrayList<String> contextWords = new ArrayList<String>();
+
+ for (int i = 0; i < sentence.length; i++) {
+
+ if (!Constants.stopWords.contains(sentence[i].toLowerCase())
+ && (wordIndex != i)) {
+
+ String word = sentence[i].toLowerCase().replaceAll("[^a-z]", "").trim();
+
+ if (!word.equals("")) {
+ String lemma = Loader.getLemmatizer().lemmatize(sentence[i],
+ posTags[i]);
+ contextWords.add(lemma);
+ }
+
+ }
+ }
+
+ return contextWords.toArray(new String[contextWords.size()]);
+ }
+
+ private String[] extractLocalCollocations(String[] sentence, int wordIndex,
+ int ngram) {
+ /*
+ * Here the author used only 11 features of this type. the range was set to
+ * 3 (bigrams extracted in a way that they are at max separated by 1 word).
+ */
+
+ ArrayList<String> localCollocations = new ArrayList<String>();
+
+ for (int i = wordIndex - ngram; i <= wordIndex + ngram; i++) {
+
+ if (!(i < 0 || i > sentence.length - 3)) {
+ if ((i != wordIndex) && (i + 1 != wordIndex)
+ && (i + 1 < wordIndex + ngram)) {
+ String lc = (sentence[i] + " " + sentence[i + 1]).toLowerCase();
+ localCollocations.add(lc);
+ }
+ if ((i != wordIndex) && (i + 2 != wordIndex)
+ && (i + 2 < wordIndex + ngram)) {
+ String lc = (sentence[i] + " " + sentence[i + 2]).toLowerCase();
+ localCollocations.add(lc);
+ }
+ }
+
+ }
+
+ String[] res = new String[localCollocations.size()];
+ res = localCollocations.toArray(res);
+
+ return res;
+ }
+
+ /**
+ * This method generates the different set of features related to the IMS
+ * approach and store them in the corresponding attributes of the WTDIMS
+ *
+ * @param word
+ * the word to disambiguate [object: WTDIMS]
+ * @param windowSize
+ * the parameter required to generate the features qualified of
+ * "PoS of Surrounding Words"
+ * @param ngram
+ * the parameter required to generate the features qualified of
+ * "Local Collocations"
+ */
+ public void extractIMSFeatures(WTDIMS word, int windowSize, int ngram) {
+
+ word.setPosOfSurroundingWords(extractPosOfSurroundingWords(
+ word.getSentence(), word.getWordIndex(), windowSize));
+ word.setSurroundingWords(extractSurroundingWords(word.getSentence(),
+ word.getWordIndex()));
+ word.setLocalCollocations(extractLocalCollocations(word.getSentence(),
+ word.getWordIndex(), ngram));
+
+ }
+
+ /**
+ * This generates the context of IMS. It supposes that the features have
+ * already been extracted and stored in the WTDIMS object, therefore it
+ * doesn't require any parameters.
+ *
+ * @param word
+ * @return the Context of the wordToDisambiguate
+ */
+ public String[] serializeIMSFeatures(WTDIMS word) {
+
+ String[] posOfSurroundingWords = word.getPosOfSurroundingWords();
+ String[] surroundingWords = word.getSurroundingWords();
+ String[] localCollocations = word.getLocalCollocations();
+
+ String[] serializedFeatures = new String[posOfSurroundingWords.length
+ + surroundingWords.length + localCollocations.length];
+
+ int i = 0;
+
+ for (String feature : posOfSurroundingWords) {
+ serializedFeatures[i] = "F" + i + "=" + feature;
+ i++;
+ }
+
+ for (String feature : surroundingWords) {
+ serializedFeatures[i] = "F" + i + "=" + feature;
+ i++;
+ }
+
+ for (String feature : localCollocations) {
+ serializedFeatures[i] = "F" + i + "=" + feature;
+ i++;
+ }
+
+ return serializedFeatures;
+
+ }
+}
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/Loader.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/Loader.java
index 9851004..127e6ff 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/Loader.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/Loader.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package opennlp.tools.disambiguator;
import java.io.File;
@@ -25,7 +44,7 @@
public class Loader {
- private static String modelsDir = "src\\test\\resources\\opennlp\\tools\\disambiguator\\";
+ private static String modelsDir = "src\\test\\resources\\models\\";
private static SentenceDetectorME sdetector;
private static Tokenizer tokenizer;
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/Node.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/Node.java
index 8f06b8c..8481d0c 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/Node.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/Node.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package opennlp.tools.disambiguator;
import java.util.ArrayList;
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/PreProcessor.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/PreProcessor.java
index c34d26b..311379c 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/PreProcessor.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/PreProcessor.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package opennlp.tools.disambiguator;
import java.util.ArrayList;
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDisambiguator.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDisambiguator.java
index 01096c9..5b67511 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDisambiguator.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDisambiguator.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package opennlp.tools.disambiguator;
import opennlp.tools.util.Span;
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordPOS.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordPOS.java
index 5eac804..cf7e31d 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordPOS.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordPOS.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package opennlp.tools.disambiguator;
import java.util.ArrayList;
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordSense.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordSense.java
index 30e6e8b..37fedce 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordSense.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordSense.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package opennlp.tools.disambiguator;
import opennlp.tools.disambiguator.lesk.WTDLesk;
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordToDisambiguate.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordToDisambiguate.java
index e572153..3eaa1ed 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordToDisambiguate.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordToDisambiguate.java
@@ -1,77 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package opennlp.tools.disambiguator;
+import java.util.ArrayList;
+
+import net.sf.extjwnl.data.POS;
+
+
+
+
public class WordToDisambiguate {
+
+ // TODO Check if it is necessary to add an attribute [word] since the word in the sentence is not necessarily in the base form ??
+
+ protected String [] sentence;
+ protected String [] posTags;
+
+ protected int wordIndex;
- protected String[] sentence;
- protected int wordIndex;
- protected String posTag;
+ protected int sense;
+
+ protected ArrayList<String> senseID;
+
+
+
+ /**
+ * Constructor
+ */
+
+
+ public WordToDisambiguate(String[] sentence, int wordIndex, int sense) throws IllegalArgumentException{
+ super();
+
+ if (wordIndex>sentence.length){
+ throw new IllegalArgumentException("The index is out of bounds !");
+ }
+
+ this.sentence = sentence;
+ this.posTags = PreProcessor.tag(sentence);
+
+ this.wordIndex = wordIndex;
+
+ this.sense = sense;
+ }
+
+ public WordToDisambiguate(String[] sentence, int wordIndex) {
+ this(sentence,wordIndex,-1);
+ }
+
+ public WordToDisambiguate() {
+ String[] emptyString = {};
+ int emptyInteger = 0;
+
+ this.sentence = emptyString;
+ this.wordIndex = emptyInteger;
+ this.sense = -1;
+
+ }
- protected int sense;
+
+ /**
+ * Getters and Setters
+ */
+
+ // Sentence
+ public String[] getSentence() {
+ return sentence;
+ }
- /**
- * Constructor
- */
+ public void setSentence(String[] sentence) {
+ this.sentence = sentence;
+ }
- public WordToDisambiguate(String[] sentence, int wordIndex, int sense)
- throws IllegalArgumentException {
- super();
+
+ // Sentence Pos-Tags
+ public String[] getPosTags() {
+ return posTags;
+ }
- if (wordIndex > sentence.length) {
- throw new IllegalArgumentException("The index is out of bounds !");
- }
- this.sentence = sentence;
- this.wordIndex = wordIndex;
- String[] posTags = PreProcessor.tag(sentence);
- this.posTag = posTags[wordIndex];
- this.sense = sense;
- }
+ public void setPosTags(String[] posTags) {
+ this.posTags = posTags;
+ }
- public WordToDisambiguate(String[] sentence, int wordIndex) {
- this(sentence, wordIndex, -1);
- }
+
+ // Word to disambiguate
+ public int getWordIndex() {
+ return wordIndex;
+ }
- /**
- * Getters and Setters
- */
+ public String getRawWord() {
+
+ /**
+ * For example, from the word "running" it returns "run.v"
+ */
+
+ String wordBaseForm = Loader.getLemmatizer().lemmatize(this.sentence[wordIndex], this.posTags[wordIndex]);
+
+ String ref = "";
+
+ if (Constants.getPOS(this.posTags[wordIndex]).equals(POS.VERB)) {
+ ref = wordBaseForm + ".v";
+ } else if (Constants.getPOS(this.posTags[wordIndex]).equals(POS.NOUN)) {
+ ref = wordBaseForm + ".n";
+ } else if (Constants.getPOS(this.posTags[wordIndex]).equals(POS.ADJECTIVE)) {
+ ref = wordBaseForm + ".a";
+ } else if (Constants.getPOS(this.posTags[wordIndex]).equals(POS.ADVERB)) {
+ ref = wordBaseForm + ".r";
+ } else {
+
+ }
+
+ return ref;
+
+ }
+
+ public String getWord() {
+ return this.sentence[this.wordIndex];
+ }
+
+ public String getPosTag() {
+ return this.posTags[this.wordIndex];
+ }
+
+ public void setWordIndex(int wordIndex) {
+ this.wordIndex = wordIndex;
+ }
+
- // sentence
- public String[] getSentence() {
- return sentence;
- }
+
+
+ // Word to disambiguate sense
+ public int getSense() {
+ return sense;
+ }
- public void setSentence(String[] sentence) {
- this.sentence = sentence;
- }
+ public void setSense(int sense) {
+ this.sense = sense;
+ }
- // word
- public int getWordIndex() {
- return wordIndex;
- }
+
+
+ // Sense as in the source
+ // TODO fix the conflict between this ID of the sense and that in the attribute [sense]
+ public ArrayList<String> getSenseID() {
+ return senseID;
+ }
- public void setWordIndex(int wordIndex) {
- this.wordIndex = wordIndex;
- }
+ public void setSenseID(ArrayList<String> senseID) {
+ this.senseID = senseID;
+ }
+
+
- public String getWord() {
- return sentence[wordIndex];
- }
- // posTag
- public String getPosTag() {
- return posTag;
- }
+ /**
+ * toString
+ */
+
+ public String toString() {
+ return (wordIndex + "\t" + getWord() + "\n" + sentence);
+ }
+
- public void setPosTag(String posTag) {
- this.posTag = posTag;
- }
+
- // sense
- public int getSense() {
- return sense;
- }
-
- public void setSense(int sense) {
- this.sense = sense;
- }
}
-
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/DefaultIMSContextGenerator.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/DefaultIMSContextGenerator.java
new file mode 100644
index 0000000..b1710d7
--- /dev/null
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/DefaultIMSContextGenerator.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package opennlp.tools.disambiguator.ims;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+
+import opennlp.tools.disambiguator.FeaturesExtractor;
+import opennlp.tools.disambiguator.ims.WTDIMS;
+
+public class DefaultIMSContextGenerator implements IMSContextGenerator {
+
+ FeaturesExtractor fExtractor = new FeaturesExtractor();
+
+ /**
+ * Default context generator for IMS.
+ */
+ public DefaultIMSContextGenerator() {
+
+ }
+
+ /**
+ * Get Context of a word To disambiguate
+ */
+ @Override
+ public String[] getContext(WTDIMS word) {
+ return fExtractor.serializeIMSFeatures(word);
+ }
+
+ /**
+ * Returns an {@link ArrayList} of features for the object of type WTDIMS
+ * Extensions of this class can override this method to create a customized
+ * {@link IMSContextGenerator}
+ *
+ * @param word
+ * : the word to disambiguate {@link WTDIMS} along with its sentence
+ * [Check the Class WTDIMS]
+ * @param numberOfSurroundingWords
+ * : the number of surrounding words used in the feature
+ * "POS Tags of Surrounding Words" Default value is 3
+ * @param ngram
+ * : the number of words used to extract the feature
+ * "Local Collocations" Default value is 2
+ *
+ * @return an {@link ArrayList} of features
+ */
+
+ protected List<String> createContext(WTDIMS word) {
+ return Arrays.asList(getContext(word));
+ }
+}
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/IMS.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/IMS.java
index 45af86d..961c6c3 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/IMS.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/IMS.java
@@ -1,20 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package opennlp.tools.disambiguator.ims;
-import java.util.ArrayList;
+import opennlp.tools.ml.maxent.GIS;
+import opennlp.tools.ml.maxent.io.GISModelReader;
+import opennlp.tools.ml.maxent.io.SuffixSensitiveGISModelWriter;
+import opennlp.tools.ml.model.AbstractModel;
+import opennlp.tools.ml.model.AbstractModelWriter;
+import opennlp.tools.ml.model.DataIndexer;
+import opennlp.tools.ml.model.DataReader;
+import opennlp.tools.ml.model.Event;
+import opennlp.tools.ml.model.OnePassDataIndexer;
+import opennlp.tools.ml.model.PlainTextFileDataReader;
-import opennlp.tools.disambiguator.WSDisambiguator;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.zip.GZIPInputStream;
+
+import opennlp.tools.disambiguator.DictionaryInstance;
+import net.sf.extjwnl.data.POS;
+import net.sf.extjwnl.data.Synset;
+import opennlp.tools.ml.model.MaxentModel;
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.ObjectStreamUtils;
import opennlp.tools.util.Span;
+import opennlp.tools.util.TrainingParameters;
+import opennlp.tools.disambiguator.Constants;
+import opennlp.tools.disambiguator.DataExtractor;
+import opennlp.tools.disambiguator.FeaturesExtractor;
+import opennlp.tools.disambiguator.PreProcessor;
+import opennlp.tools.disambiguator.WordPOS;
+import opennlp.tools.disambiguator.WSDisambiguator;
public class IMS implements WSDisambiguator {
- FeaturesExtractor fExtractor = new FeaturesExtractor();
+ // private MaxentModel model;
+ private IMSFactory factory;
+
+ private final IMSContextGenerator cg;
+
+ private FeaturesExtractor fExtractor = new FeaturesExtractor();
+ private DataExtractor dExtractor = new DataExtractor();
/**
* PARAMETERS
*/
- int numberOfSurroundingWords;
- int ngram;
+ private int windowSize;
+ private int ngram;
/**
* Constructors
@@ -22,67 +77,264 @@
public IMS() {
super();
- numberOfSurroundingWords = 3;
+ windowSize = 3;
ngram = 2;
+
+ IMSFactory factory = new IMSFactory();
+ this.factory = factory;
+ this.cg = factory.createContextGenerator();
}
- public IMS(int numberOfSurroundingWords, int ngram) {
+ public IMS(int windowSize, int ngram) {
super();
- this.numberOfSurroundingWords = numberOfSurroundingWords;
+ this.windowSize = windowSize;
this.ngram = ngram;
+
+ IMSFactory factory = new IMSFactory();
+ this.factory = factory;
+ this.cg = factory.createContextGenerator();
}
/**
* INTERNAL METHODS
*/
- private void extractFeature(ArrayList<WTDIMS> words) {
+ protected HashMap<Integer, WTDIMS> extractTrainingData(
+ String wordTrainingxmlFile,
+ HashMap<String, ArrayList<DictionaryInstance>> senses) {
- for (WTDIMS word : words) {
+ /**
+ * word tag has to be in the format "word.t" (e.g., "activate.v", "smart.a",
+ * etc.)
+ */
- word.setPosOfSurroundingWords(fExtractor.extractPosOfSurroundingWords(
- word.getSentence(), word.getWordIndex(), numberOfSurroundingWords));
+ HashMap<Integer, WTDIMS> trainingData = dExtractor
+ .extractWSDInstances(wordTrainingxmlFile);
- word.setSurroundingWords(fExtractor.extractSurroundingWords(
- word.getSentence(), word.getWordIndex()));
+ // HashMap<Integer, WTDIMS> trainingData =
+ // dExtractor.extractWSDInstances(wordTrainingxmlFile);
- word.setLocalCollocations(fExtractor.extractLocalCollocations(
- word.getSentence(), word.getWordIndex(), ngram));
+ for (Integer key : trainingData.keySet()) {
+ for (String senseId : trainingData.get(key).getSenseID()) {
+ for (String dictKey : senses.keySet()) {
+ for (DictionaryInstance instance : senses.get(dictKey)) {
+ if (senseId.equals(instance.getId())) {
+ trainingData.get(key).setSense(
+ Integer.parseInt(dictKey.split("_")[1]));
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ return trainingData;
+ }
+
+ protected void extractFeature(HashMap<Integer, WTDIMS> words) {
+
+ for (Integer key : words.keySet()) {
+
+ fExtractor.extractIMSFeatures(words.get(key), windowSize, ngram);
}
}
- private ArrayList<WTDIMS> extractTrainingData(String xmlFile) {
+ protected String getTrainingFile(WTDIMS wtd) {
- ArrayList<WTDIMS> trainingData = new ArrayList<WTDIMS>();
+ String wordBaseForm = PreProcessor
+ .lemmatize(wtd.getWord(), wtd.getPosTag());
- /**
- * TODO Processing of the xml File here (To check the format of the data)
- */
+ String ref = "";
- return trainingData;
+ if (Constants.getPOS(wtd.getPosTag()).equals(POS.VERB)) {
+ ref = wordBaseForm + ".v";
+ } else if (Constants.getPOS(wtd.getPosTag()).equals(POS.NOUN)) {
+ ref = wordBaseForm + ".n";
+ } else if (Constants.getPOS(wtd.getPosTag()).equals(POS.ADJECTIVE)) {
+ ref = wordBaseForm + ".a";
+ } else if (Constants.getPOS(wtd.getPosTag()).equals(POS.ADVERB)) {
+ ref = wordBaseForm + ".r";
+ } else {
+
+ }
+
+ return ref;
}
- public void train(String trainingSetFile) { // TODO To revise after finihsing
- // the implementation of the
- // collector
+ protected HashMap<String, String> getWordDictionaryInstance(WTDIMS wtd) {
- ArrayList<WTDIMS> instances = extractTrainingData(trainingSetFile);
+ String dict = factory.getDict();
+ String map = factory.getMap();
- extractFeature(instances);
+ return dExtractor.getDictionaryInstance(dict, map,
+ this.getTrainingFile(wtd));
}
- public void load(String binFile) {
- // TODO After finishing training the training data
+ protected String[] getMostFrequentSense(WTDIMS wordToDisambiguate) {
+ String word = wordToDisambiguate.getRawWord();
+ POS pos = Constants.getPOS(wordToDisambiguate.getPosTag());
+
+ WordPOS wordPOS = new WordPOS(word, pos);
+
+ ArrayList<Synset> synsets = wordPOS.getSynsets();
+
+ int size = synsets.size();
+
+ String[] senses = new String[size];
+
+ for (int i = 0; i < size; i++) {
+ senses[i] = synsets.get(i).getGloss();
+ }
+
+ return senses;
+
+ }
+
+ /**
+ * PUBLIC METHODS
+ */
+
+ public void train(String wordTag, TrainingParameters trainParams) {
+
+ String rawDataDirectory = factory.getRawDataDirectory();
+ String trainingDataDirectory = factory.getTrainingDataDirectory();
+ String dict = factory.getDict();
+ String map = factory.getMap();
+
+ String wordTrainingxmlFile = rawDataDirectory + wordTag + ".xml";
+ String wordTrainingbinFile = trainingDataDirectory + wordTag + ".gz";
+
+ File bf = new File(wordTrainingxmlFile);
+
+ ObjectStream IMSes = null;
+
+ if (bf.exists() && !bf.isDirectory()) {
+
+ HashMap<String, ArrayList<DictionaryInstance>> senses = dExtractor
+ .extractWordSenses(dict, map, wordTag);
+
+ HashMap<Integer, WTDIMS> instances = extractTrainingData(
+ wordTrainingxmlFile, senses);
+
+ extractFeature(instances);
+
+ ArrayList<Event> events = new ArrayList<Event>();
+
+ for (int key : instances.keySet()) {
+
+ int sense = instances.get(key).getSense();
+
+ String[] context = cg.getContext(instances.get(key));
+
+ Event ev = new Event(sense + "", context);
+
+ events.add(ev);
+
+ // Collection collEvents = events;
+
+ IMSes = ObjectStreamUtils.createObjectStream(events);
+
+ }
+
+ DataIndexer indexer;
+ try {
+ indexer = new OnePassDataIndexer((ObjectStream<Event>) IMSes);
+ MaxentModel trainedMaxentModel = GIS.trainModel(100, indexer);
+ File outFile = new File(wordTrainingbinFile);
+ AbstractModelWriter writer = new SuffixSensitiveGISModelWriter(
+ (AbstractModel) trainedMaxentModel, outFile);
+ writer.persist();
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+ }
+
+ public MaxentModel load(String binFile) {
+
+ MaxentModel loadedMaxentModel = null;
+
+ FileInputStream inputStream;
+ try {
+ inputStream = new FileInputStream(binFile);
+ InputStream decodedInputStream = new GZIPInputStream(inputStream);
+ DataReader modelReader = new PlainTextFileDataReader(decodedInputStream);
+ loadedMaxentModel = new GISModelReader(modelReader).getModel();
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return loadedMaxentModel;
}
@Override
public String[] disambiguate(String[] inputText, int inputWordIndex) {
- // TODO Auto-generated method stub
- return null;
+
+ String rawDataDirectory = factory.getRawDataDirectory();
+ String trainingDataDirectory = factory.getTrainingDataDirectory();
+
+ WTDIMS word = new WTDIMS(inputText, inputWordIndex);
+ fExtractor.extractIMSFeatures(word, windowSize, ngram);
+
+ String wordTag = getTrainingFile(word);
+
+ String wordTrainingxmlFile = rawDataDirectory + wordTag + ".xml";
+ String wordTrainingbinFile = trainingDataDirectory + wordTag + ".gz";
+
+ File bf = new File(wordTrainingbinFile);
+
+ MaxentModel loadedMaxentModel = null;
+ String outcome = "";
+ if (bf.exists() && !bf.isDirectory()) {
+ // if the model file exists already
+ // System.out.println("the model file was found !");
+ loadedMaxentModel = load(wordTrainingbinFile);
+ String[] context = cg.getContext(word);
+
+ double[] outcomeProbs = loadedMaxentModel.eval(context);
+ outcome = loadedMaxentModel.getBestOutcome(outcomeProbs);
+
+ } else {
+ bf = new File(wordTrainingxmlFile);
+ if (bf.exists() && !bf.isDirectory()) {
+ // if the xml file exists already
+ // System.out.println("the xml file was found !");
+ train(wordTag, null);
+ bf = new File(wordTrainingbinFile);
+ loadedMaxentModel = load(wordTrainingbinFile);
+ String[] context = cg.getContext(word);
+
+ double[] outcomeProbs = loadedMaxentModel.eval(context);
+ outcome = loadedMaxentModel.getBestOutcome(outcomeProbs);
+ }
+ }
+
+ if (!outcome.equals("")) {
+
+ HashMap<String, String> senses = getWordDictionaryInstance(word);
+
+ String index = wordTag + "_" + outcome;
+
+ String[] s = { senses.get(index) };
+
+ return s;
+
+ } else {
+ // if no training data exist
+ // System.out.println("No training data available, the MFS is returned !");
+ String[] s = getMostFrequentSense(word);
+ return s;
+ }
+
}
@Override
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/IMSContextGenerator.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/IMSContextGenerator.java
new file mode 100644
index 0000000..7b88474
--- /dev/null
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/IMSContextGenerator.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package opennlp.tools.disambiguator.ims;
+
+/**
+ * Interface for {@link IMSME} context generators.
+ */
+public interface IMSContextGenerator {
+
+ public String[] getContext(WTDIMS word);
+
+}
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/IMSEventStream.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/IMSEventStream.java
new file mode 100644
index 0000000..f35b4cc
--- /dev/null
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/IMSEventStream.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package opennlp.tools.disambiguator.ims;
+
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import opennlp.tools.ml.model.Event;
+import opennlp.tools.util.AbstractEventStream;
+import opennlp.tools.util.ObjectStream;
+
+public class IMSEventStream extends AbstractEventStream<WTDIMS> {
+
+ private IMSContextGenerator cg;
+
+ public IMSEventStream(ObjectStream<WTDIMS> samples) {
+ super(samples);
+ }
+
+ @Override
+ protected Iterator<Event> createEvents(WTDIMS sample) {
+ List<Event> events = new ArrayList<Event>();
+
+ int sense = sample.getSense();
+
+ String[] context = cg.getContext(sample);
+
+ Event ev = new Event(sense + "", context);
+
+ events.add(ev);
+
+ return events.iterator();
+ }
+
+}
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/IMSFactory.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/IMSFactory.java
new file mode 100644
index 0000000..5c078f6
--- /dev/null
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/IMSFactory.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package opennlp.tools.disambiguator.ims;
+
+import opennlp.tools.util.BaseToolFactory;
+import opennlp.tools.util.InvalidFormatException;
+
+public class IMSFactory extends BaseToolFactory {
+
+ protected String languageCode;
+
+ protected String resourcesFolder = "src\\test\\resources\\supervised\\";
+
+ protected String rawDataDirectory = resourcesFolder + "training\\";
+ protected String trainingDataDirectory = resourcesFolder + "models\\";
+ protected String dictionaryDirectory = resourcesFolder + "dictionary\\";
+
+ protected String dict = dictionaryDirectory + "EnglishLS.dictionary.xml";
+ protected String map = dictionaryDirectory + "EnglishLS.sensemap";
+
+ public IMSFactory() {
+ super();
+ }
+
+ public String getLanguageCode() {
+ return languageCode;
+ }
+
+ public void setLanguageCode(String languageCode) {
+ this.languageCode = languageCode;
+ }
+
+ public String getRawDataDirectory() {
+ return rawDataDirectory;
+ }
+
+ public void setRawDataDirectory(String rawDataDirectory) {
+ this.rawDataDirectory = rawDataDirectory;
+ }
+
+ public String getTrainingDataDirectory() {
+ return trainingDataDirectory;
+ }
+
+ public void setTrainingDataDirectory(String trainingDataDirectory) {
+ this.trainingDataDirectory = trainingDataDirectory;
+ }
+
+ public String getDictionaryDirectory() {
+ return dictionaryDirectory;
+ }
+
+ public void setDictionaryDirectory(String dictionaryDirectory) {
+ this.dictionaryDirectory = dictionaryDirectory;
+ }
+
+ public String getDict() {
+ return dict;
+ }
+
+ public void setDict(String dict) {
+ this.dict = dict;
+ }
+
+ public String getMap() {
+ return map;
+ }
+
+ public void setMap(String map) {
+ this.map = map;
+ }
+
+ void init() {
+ }
+
+ public IMSContextGenerator createContextGenerator() {
+
+ return new DefaultIMSContextGenerator();
+ }
+
+ @Override
+ public void validateArtifactMap() throws InvalidFormatException {
+ }
+}
+
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/WTDIMS.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/WTDIMS.java
index dcb9f06..ec9fd10 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/WTDIMS.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/ims/WTDIMS.java
@@ -1,25 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package opennlp.tools.disambiguator.ims;
+
+
import java.util.ArrayList;
+import opennlp.tools.disambiguator.PreProcessor;
import opennlp.tools.disambiguator.WordToDisambiguate;
public class WTDIMS extends WordToDisambiguate {
protected String[] posOfSurroundingWords;
protected String[] surroundingWords;
- protected ArrayList<String[]> localCollocations;
+ protected String[] localCollocations;
- /**
- * Constructor
- */
public WTDIMS(String[] sentence, int word, int sense) {
super(sentence, word, sense);
+
}
- /**
- * Getters and Setters
- */
+ public WTDIMS(String[] sentence, int word) {
+ super(sentence, word);
+ }
+
+ public WTDIMS(String xmlWord, ArrayList<String> xmlAnswers,
+ String xmlSentence, String xmlrawWord) {
+ super();
+
+ // this.word = xmlWord;
+
+ this.sentence = PreProcessor.tokenize(xmlSentence);
+ this.posTags = PreProcessor.tag(this.sentence);
+
+ for (int i = 0; i < sentence.length; i++) {
+ if (xmlrawWord.equals(sentence[i])) {
+ this.wordIndex = i;
+ break;
+ }
+ }
+
+ this.senseID = xmlAnswers;
+
+ }
public String[] getPosOfSurroundingWords() {
return posOfSurroundingWords;
@@ -37,13 +77,12 @@
this.surroundingWords = surroundingWords;
}
- public ArrayList<String[]> getLocalCollocations() {
+ public String[] getLocalCollocations() {
return localCollocations;
}
- public void setLocalCollocations(ArrayList<String[]> localCollocations) {
+ public void setLocalCollocations(String[] localCollocations) {
this.localCollocations = localCollocations;
}
-
}
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/lesk/Lesk.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/lesk/Lesk.java
index cdc94d2..ecb1ec0 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/lesk/Lesk.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/lesk/Lesk.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package opennlp.tools.disambiguator.lesk;
import java.security.InvalidParameterException;
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/lesk/LeskParameters.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/lesk/LeskParameters.java
index 8618795..3d1834a 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/lesk/LeskParameters.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/lesk/LeskParameters.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package opennlp.tools.disambiguator.lesk;
public class LeskParameters {
@@ -82,4 +101,3 @@
}
}
-
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/lesk/WTDLesk.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/lesk/WTDLesk.java
index c871928..6f7a37a 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/lesk/WTDLesk.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/lesk/WTDLesk.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package opennlp.tools.disambiguator.lesk;
import opennlp.tools.disambiguator.WordToDisambiguate;
diff --git a/opennlp-wsd/src/test/java/opennlp/tools/disambiguator/IMSTester.java b/opennlp-wsd/src/test/java/opennlp/tools/disambiguator/IMSTester.java
new file mode 100644
index 0000000..4717275
--- /dev/null
+++ b/opennlp-wsd/src/test/java/opennlp/tools/disambiguator/IMSTester.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package opennlp.tools.disambiguator;
+
+import opennlp.tools.disambiguator.ims.IMS;
+
+public class IMSTester {
+
+ public static void main(String[] args) {
+
+ IMS ims = new IMS();
+
+ String test = "You have to write an essay without using a dictionary!";
+ String[] sentence = Loader.getTokenizer().tokenize(test);
+ Constants.print(ims.disambiguate(sentence, 3));
+
+ String test2 = "Please write to me soon.";
+ String[] sentence2 = Loader.getTokenizer().tokenize(test2);
+ Constants.print(ims.disambiguate(sentence2, 1));
+
+ String test3 = "the argument over foreign aid goes on and on";
+ String[] sentence3 = Loader.getTokenizer().tokenize(test3);
+ Constants.print(ims.disambiguate(sentence3, 1));
+
+ String test4 = "it was a strong argument that his hypothesis was true";
+ String[] sentence4 = Loader.getTokenizer().tokenize(test4);
+ Constants.print(ims.disambiguate(sentence4, 3));
+
+ }
+
+}
diff --git a/opennlp-wsd/src/test/java/opennlp/tools/disambiguator/Tester.java b/opennlp-wsd/src/test/java/opennlp/tools/disambiguator/Tester.java
index e38f749..c78ba80 100644
--- a/opennlp-wsd/src/test/java/opennlp/tools/disambiguator/Tester.java
+++ b/opennlp-wsd/src/test/java/opennlp/tools/disambiguator/Tester.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package opennlp.tools.disambiguator;
import java.io.File;
