Merge pull request #1 from kojisekig/OPENNLP-1201
OPENNLP-1201: add auxiliary info support to token in TokenNameFinder.…
diff --git a/japanese-addon/pom.xml b/japanese-addon/pom.xml
new file mode 100644
index 0000000..8ceff3e
--- /dev/null
+++ b/japanese-addon/pom.xml
@@ -0,0 +1,31 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>japanese-addon</artifactId>
+ <packaging>jar</packaging>
+ <version>1.0-SNAPSHOT</version>
+ <name>japanese-addon</name>
+ <url>http://maven.apache.org</url>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-tools</artifactId>
+ <version>1.9.1-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/japanese-addon/src/main/java/opennlp/tools/namefind/AuxiliaryInfoNameContextGenerator.java b/japanese-addon/src/main/java/opennlp/tools/namefind/AuxiliaryInfoNameContextGenerator.java
new file mode 100644
index 0000000..a45a7c7
--- /dev/null
+++ b/japanese-addon/src/main/java/opennlp/tools/namefind/AuxiliaryInfoNameContextGenerator.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.namefind;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import opennlp.tools.util.featuregen.AdaptiveFeatureGenerator;
+import opennlp.tools.util.featuregen.FeatureGeneratorUtil;
+
+/**
+ * If a token contains an auxiliary information, e.g. POS tag, this class can be used
+ * to extract word part in {@link #getContext(int, String[], String[], Object[])} method.
+ *
+ * <strong>EXPERIMENTAL</strong>.
+ * This class has been added as part of a work in progress and might change without notice.
+ */
+public class AuxiliaryInfoNameContextGenerator extends DefaultNameContextGenerator {
+
+ public AuxiliaryInfoNameContextGenerator(AdaptiveFeatureGenerator... featureGenerators) {
+ super(featureGenerators);
+ }
+
+ /**
+ * Return the context for finding names at the specified index.
+ * @param index The index of the token in the specified toks array for which the
+ * context should be constructed.
+ * @param tokens The tokens of the sentence. The <code>toString</code> methods
+ * of these objects should return the token text.
+ * @param preds The previous decisions made in the tagging of this sequence.
+ * Only indices less than i will be examined.
+ * @param additionalContext Addition features which may be based on a context outside of the sentence.
+ *
+ * @return the context for finding names at the specified index.
+ */
+ @Override
+ public String[] getContext(int index, String[] tokens, String[] preds, Object[] additionalContext) {
+ List<String> features = new ArrayList<>();
+
+ for (AdaptiveFeatureGenerator featureGenerator : featureGenerators) {
+ featureGenerator.createFeatures(features, tokens, index, preds);
+ }
+
+ //previous outcome features
+ String po = NameFinderME.OTHER;
+ String ppo = NameFinderME.OTHER;
+
+ // TODO: These should be moved out here in its own feature generator!
+ if (preds != null) {
+ if (index > 1) {
+ ppo = preds[index - 2];
+ }
+
+ if (index > 0) {
+ po = preds[index - 1];
+ }
+ features.add("po=" + po);
+ String word = AuxiliaryInfoUtil.getWordPart(tokens[index]);
+ features.add("pow=" + po + "," + word);
+ features.add("powf=" + po + "," + FeatureGeneratorUtil.tokenFeature(word));
+ features.add("ppo=" + ppo);
+ }
+
+ return features.toArray(new String[features.size()]);
+ }
+}
diff --git a/japanese-addon/src/main/java/opennlp/tools/namefind/AuxiliaryInfoTokenNameFinderFactory.java b/japanese-addon/src/main/java/opennlp/tools/namefind/AuxiliaryInfoTokenNameFinderFactory.java
new file mode 100644
index 0000000..ea091e6
--- /dev/null
+++ b/japanese-addon/src/main/java/opennlp/tools/namefind/AuxiliaryInfoTokenNameFinderFactory.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.namefind;
+
+import opennlp.tools.util.featuregen.AdaptiveFeatureGenerator;
+import opennlp.tools.util.featuregen.BigramNameFeatureGenerator;
+import opennlp.tools.util.featuregen.CachedFeatureGenerator;
+import opennlp.tools.util.featuregen.OutcomePriorFeatureGenerator;
+import opennlp.tools.util.featuregen.PreviousMapFeatureGenerator;
+import opennlp.tools.util.featuregen.SentenceFeatureGenerator;
+import opennlp.tools.util.featuregen.TokenClassFeatureGenerator;
+import opennlp.tools.util.featuregen.TokenFeatureGenerator;
+import opennlp.tools.util.featuregen.WindowFeatureGenerator;
+
+/**
+ * If a token contains an auxiliary information, e.g. POS tag, in the training data,
+ * you can use this class via -factory command line option.
+ *
+ * <strong>EXPERIMENTAL</strong>.
+ * This class has been added as part of a work in progress and might change without notice.
+ */
+public class AuxiliaryInfoTokenNameFinderFactory extends TokenNameFinderFactory {
+
+ @Override
+ public NameContextGenerator createContextGenerator() {
+
+ AdaptiveFeatureGenerator featureGenerator = createFeatureGenerators();
+
+ if (featureGenerator == null) {
+ featureGenerator = new CachedFeatureGenerator(
+ new WindowFeatureGenerator(new TokenFeatureGenerator(), 2, 2),
+ new WindowFeatureGenerator(new TokenClassFeatureGenerator(true), 2, 2),
+ new OutcomePriorFeatureGenerator(),
+ new PreviousMapFeatureGenerator(),
+ new BigramNameFeatureGenerator(),
+ new SentenceFeatureGenerator(true, false));
+ }
+
+ return new AuxiliaryInfoNameContextGenerator(featureGenerator);
+ }
+}
diff --git a/japanese-addon/src/main/java/opennlp/tools/namefind/AuxiliaryInfoUtil.java b/japanese-addon/src/main/java/opennlp/tools/namefind/AuxiliaryInfoUtil.java
new file mode 100644
index 0000000..7bb336c
--- /dev/null
+++ b/japanese-addon/src/main/java/opennlp/tools/namefind/AuxiliaryInfoUtil.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.namefind;
+
+/**
+ * If a token contains an auxiliary information, e.g. POS tag, this class can be used
+ * to extract word part or auxiliary information part.<br>
+ *
+ * ex) token := word '/' POStag
+ *
+ * <strong>EXPERIMENTAL</strong>.
+ * This class has been added as part of a work in progress and might change without notice.
+ */
+public class AuxiliaryInfoUtil {
+
+ public static int getSeparatorIndex(String wordAndAux) {
+ int idx = wordAndAux.lastIndexOf('/');
+ if (idx < 0)
+ throw new RuntimeException(String.format("token %s doesn't have Auxiliary info", wordAndAux));
+ return idx;
+ }
+
+ public static String getWordPart(String wordAndAux) {
+ int idx = getSeparatorIndex(wordAndAux);
+ return idx == 0 ? " " : wordAndAux.substring(0, idx);
+ }
+
+ public static String[] getWordParts(String[] wordAndAuxes) {
+ String[] results = new String[wordAndAuxes.length];
+ for (int i = 0; i < wordAndAuxes.length; i++) {
+ results[i] = getWordPart(wordAndAuxes[i]);
+ }
+ return results;
+ }
+
+ public static String getAuxPart(String wordAndAux) {
+ int idx = getSeparatorIndex(wordAndAux);
+ return wordAndAux.substring(idx + 1);
+ }
+
+ public static String[] getAuxParts(String[] wordAndAuxes) {
+ String[] results = new String[wordAndAuxes.length];
+ for (int i = 0; i < wordAndAuxes.length; i++) {
+ results[i] = getAuxPart(wordAndAuxes[i]);
+ }
+ return results;
+ }
+}
diff --git a/japanese-addon/src/main/java/opennlp/tools/util/featuregen/AuxiliaryInfoAwareDelegateFeatureGenerator.java b/japanese-addon/src/main/java/opennlp/tools/util/featuregen/AuxiliaryInfoAwareDelegateFeatureGenerator.java
new file mode 100644
index 0000000..ee82c0c
--- /dev/null
+++ b/japanese-addon/src/main/java/opennlp/tools/util/featuregen/AuxiliaryInfoAwareDelegateFeatureGenerator.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util.featuregen;
+
+import java.util.Arrays;
+import java.util.List;
+
+import opennlp.tools.namefind.AuxiliaryInfoUtil;
+
+/**
+ * If a token contains an auxiliary information, e.g. POS tag, in the training data,
+ * you can use this feature generator in order to let the feature generator choose
+ * word part or auxiliary information part.<br>
+ *
+ * ex) token := word '/' POStag
+ *
+ * <strong>EXPERIMENTAL</strong>.
+ * This class has been added as part of a work in progress and might change without notice.
+ */
+public class AuxiliaryInfoAwareDelegateFeatureGenerator implements AdaptiveFeatureGenerator {
+
+ private String[] cachedWordAndAuxes;
+ private String[] cached;
+
+ private final AdaptiveFeatureGenerator generator;
+ private final boolean useAux;
+
+ public AuxiliaryInfoAwareDelegateFeatureGenerator(AdaptiveFeatureGenerator generator, boolean useAux) {
+ this.generator = generator;
+ this.useAux = useAux;
+ }
+
+ @Override
+ public void createFeatures(List<String> features, String[] wordAndAuxes, int index,
+ String[] previousOutcomes) {
+ if (useAux) {
+ if (!Arrays.equals(cachedWordAndAuxes, wordAndAuxes)) {
+ cachedWordAndAuxes = wordAndAuxes;
+ cached = AuxiliaryInfoUtil.getAuxParts(wordAndAuxes);
+ }
+ }
+ else {
+ if (!Arrays.equals(cachedWordAndAuxes, wordAndAuxes)) {
+ cachedWordAndAuxes = wordAndAuxes;
+ cached = AuxiliaryInfoUtil.getWordParts(wordAndAuxes);
+ }
+ }
+ generator.createFeatures(features, cached, index, previousOutcomes);
+ }
+}
diff --git a/japanese-addon/src/main/java/opennlp/tools/util/featuregen/AuxiliaryInfoAwareDelegateFeatureGeneratorFactory.java b/japanese-addon/src/main/java/opennlp/tools/util/featuregen/AuxiliaryInfoAwareDelegateFeatureGeneratorFactory.java
new file mode 100644
index 0000000..28d71e4
--- /dev/null
+++ b/japanese-addon/src/main/java/opennlp/tools/util/featuregen/AuxiliaryInfoAwareDelegateFeatureGeneratorFactory.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util.featuregen;
+
+import opennlp.tools.util.InvalidFormatException;
+
+/**
+ * @see opennlp.tools.util.featuregen.AuxiliaryInfoAwareDelegateFeatureGenerator
+ *
+ * <strong>EXPERIMENTAL</strong>.
+ * This class has been added as part of a work in progress and might change without notice.
+ */
+public class AuxiliaryInfoAwareDelegateFeatureGeneratorFactory
+ extends GeneratorFactory.AbstractXmlFeatureGeneratorFactory {
+
+ @Override
+ public AdaptiveFeatureGenerator create() throws InvalidFormatException {
+ AdaptiveFeatureGenerator generator = (AdaptiveFeatureGenerator)args.get("generator#0");
+ if (generator == null) {
+ throw new InvalidFormatException("AuxiliaryInfoAwareDelegate feature generator must contain" +
+ " an aggregator element");
+ }
+ return new AuxiliaryInfoAwareDelegateFeatureGenerator(generator, getBool("useAux", false));
+ }
+}
diff --git a/japanese-addon/src/main/resources/opennlp/tools/namefind/ner-auxinfo-features.xml b/japanese-addon/src/main/resources/opennlp/tools/namefind/ner-auxinfo-features.xml
new file mode 100644
index 0000000..9344997
--- /dev/null
+++ b/japanese-addon/src/main/resources/opennlp/tools/namefind/ner-auxinfo-features.xml
@@ -0,0 +1,59 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<!-- [EXPERIMENTAL] Auxiliary information name finder feature generator configuration -->
+<featureGenerators cache="true" name="nameFinder">
+ <generator class="opennlp.tools.util.featuregen.AuxiliaryInfoAwareDelegateFeatureGeneratorFactory">
+ <generator class="opennlp.tools.util.featuregen.WindowFeatureGeneratorFactory">
+ <int name="prevLength">2</int>
+ <int name="nextLength">2</int>
+ <generator class="opennlp.tools.util.featuregen.TokenClassFeatureGeneratorFactory"/>
+ </generator>
+ </generator>
+ <generator class="opennlp.tools.util.featuregen.AuxiliaryInfoAwareDelegateFeatureGeneratorFactory">
+ <generator class="opennlp.tools.util.featuregen.WindowFeatureGeneratorFactory">
+ <int name="prevLength">2</int>
+ <int name="nextLength">2</int>
+ <generator class="opennlp.tools.util.featuregen.TokenFeatureGeneratorFactory"/>
+ </generator>
+ </generator>
+ <generator class="opennlp.tools.util.featuregen.AuxiliaryInfoAwareDelegateFeatureGeneratorFactory">
+ <bool name="useAux">true</bool>
+ <generator class="opennlp.tools.util.featuregen.WindowFeatureGeneratorFactory">
+ <int name="prevLength">2</int>
+ <int name="nextLength">2</int>
+ <generator class="opennlp.tools.util.featuregen.TokenFeatureGeneratorFactory"/>
+ </generator>
+ </generator>
+ <generator class="opennlp.tools.util.featuregen.AuxiliaryInfoAwareDelegateFeatureGeneratorFactory">
+ <generator class="opennlp.tools.util.featuregen.DefinitionFeatureGeneratorFactory"/>
+ </generator>
+ <generator class="opennlp.tools.util.featuregen.AuxiliaryInfoAwareDelegateFeatureGeneratorFactory">
+ <generator class="opennlp.tools.util.featuregen.PreviousMapFeatureGeneratorFactory"/>
+ </generator>
+ <generator class="opennlp.tools.util.featuregen.AuxiliaryInfoAwareDelegateFeatureGeneratorFactory">
+ <generator class="opennlp.tools.util.featuregen.BigramNameFeatureGeneratorFactory"/>
+ </generator>
+ <generator class="opennlp.tools.util.featuregen.AuxiliaryInfoAwareDelegateFeatureGeneratorFactory">
+ <generator class="opennlp.tools.util.featuregen.SentenceFeatureGeneratorFactory">
+ <bool name="begin">true</bool>
+ <bool name="end">false</bool>
+ </generator>
+ </generator>
+</featureGenerators>
diff --git a/japanese-addon/src/test/java/opennlp/tools/namefind/AuxiliaryInfoUtilTest.java b/japanese-addon/src/test/java/opennlp/tools/namefind/AuxiliaryInfoUtilTest.java
new file mode 100644
index 0000000..91bd756
--- /dev/null
+++ b/japanese-addon/src/test/java/opennlp/tools/namefind/AuxiliaryInfoUtilTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.namefind;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class AuxiliaryInfoUtilTest {
+
+ @Test
+ public void testGetSeparatorIndex() throws Exception {
+ Assert.assertEquals(0, AuxiliaryInfoUtil.getSeparatorIndex("/POStag"));
+ Assert.assertEquals(1, AuxiliaryInfoUtil.getSeparatorIndex("1/POStag"));
+ Assert.assertEquals(10, AuxiliaryInfoUtil.getSeparatorIndex("word/stuff/POStag"));
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testGetSeparatorIndexNoPos() throws Exception {
+ AuxiliaryInfoUtil.getSeparatorIndex("NOPOStags");
+ }
+
+ @Test
+ public void testGetWordPart() throws Exception {
+ Assert.assertEquals(" ", AuxiliaryInfoUtil.getWordPart("/POStag"));
+ Assert.assertEquals("1", AuxiliaryInfoUtil.getWordPart("1/POStag"));
+ Assert.assertEquals("word", AuxiliaryInfoUtil.getWordPart("word/POStag"));
+ Assert.assertEquals("word/stuff", AuxiliaryInfoUtil.getWordPart("word/stuff/POStag"));
+ }
+
+ @Test
+ public void testGetWordParts() throws Exception {
+ String[] results = AuxiliaryInfoUtil.getWordParts(new String[]{"1/A", "234/B", "3456/C", "/D"});
+ Assert.assertEquals(4, results.length);
+ Assert.assertEquals("1", results[0]);
+ Assert.assertEquals("234", results[1]);
+ Assert.assertEquals("3456", results[2]);
+ Assert.assertEquals(" ", results[3]);
+ }
+
+ @Test
+ public void testGetAuxPart() throws Exception {
+ Assert.assertEquals("POStag", AuxiliaryInfoUtil.getAuxPart("/POStag"));
+ Assert.assertEquals("POStag", AuxiliaryInfoUtil.getAuxPart("1/POStag"));
+ Assert.assertEquals("POStag", AuxiliaryInfoUtil.getAuxPart("word/POStag"));
+ Assert.assertEquals("POStag", AuxiliaryInfoUtil.getAuxPart("word/stuff/POStag"));
+ }
+
+ @Test
+ public void testGetAuxParts() throws Exception {
+ String[] results = AuxiliaryInfoUtil.getAuxParts(new String[] {"1/ABC", "234/B", "3456/CD", "/DEFGH"});
+ Assert.assertEquals(4, results.length);
+ Assert.assertEquals("ABC", results[0]);
+ Assert.assertEquals("B", results[1]);
+ Assert.assertEquals("CD", results[2]);
+ Assert.assertEquals("DEFGH", results[3]);
+ }
+}
diff --git a/japanese-addon/src/test/java/opennlp/tools/util/featuregen/AuxiliaryInfoAwareDelegateFeatureGeneratorTest.java b/japanese-addon/src/test/java/opennlp/tools/util/featuregen/AuxiliaryInfoAwareDelegateFeatureGeneratorTest.java
new file mode 100644
index 0000000..aa03110
--- /dev/null
+++ b/japanese-addon/src/test/java/opennlp/tools/util/featuregen/AuxiliaryInfoAwareDelegateFeatureGeneratorTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util.featuregen;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class AuxiliaryInfoAwareDelegateFeatureGeneratorTest {
+
+ private final String[] testSentence = "w1/pos1 w2/pos2 w3/pos3 w4/pos4".split("\\s+");
+
+ private List<String> features;
+
+ @Before
+ public void setUp() throws Exception {
+ features = new ArrayList<>();
+ }
+
+ @Test
+ public void testWord() throws Exception {
+ AdaptiveFeatureGenerator featureGenerator = new AuxiliaryInfoAwareDelegateFeatureGenerator(
+ new IdentityFeatureGenerator(), false);
+
+ featureGenerator.createFeatures(features, testSentence, 2, null);
+ Assert.assertEquals(1, features.size());
+ Assert.assertEquals("w3", features.get(0));
+ }
+
+ @Test
+ public void testAuxInfo() throws Exception {
+ AdaptiveFeatureGenerator featureGenerator = new AuxiliaryInfoAwareDelegateFeatureGenerator(
+ new IdentityFeatureGenerator(), true);
+
+ featureGenerator.createFeatures(features, testSentence, 3, null);
+ Assert.assertEquals(1, features.size());
+ Assert.assertEquals("pos4", features.get(0));
+ }
+
+ static class IdentityFeatureGenerator implements AdaptiveFeatureGenerator {
+
+ @Override
+ public void createFeatures(List<String> features, String[] tokens, int index, String[] previousOutcomes) {
+ features.add(tokens[index]);
+ }
+ }
+}
