commit c136c857087232f269f0f4c7610ab02a4129caa7
author Jörn Kottmann <joern@apache.org> Thu May 31 11:45:41 2018 +0200
committer Jörn Kottmann <joern@apache.org> Thu May 31 11:45:41 2018 +0200
tree f4f918f2eccb7e2c9c399724a5115f4adcd17440
parent 8dc9495d78017eb24cc7f6cf120c14d0c4964a3c

Adjust settings to match namefinder.py trainer

tf-ner-poc/src/main/java/org/apache/opennlp/namefinder/WordIndexer.java

diff --git a/tf-ner-poc/src/main/java/org/apache/opennlp/namefinder/WordIndexer.java b/tf-ner-poc/src/main/java/org/apache/opennlp/namefinder/WordIndexer.java
index 3954092..738a952 100644
--- a/tf-ner-poc/src/main/java/org/apache/opennlp/namefinder/WordIndexer.java
+++ b/tf-ner-poc/src/main/java/org/apache/opennlp/namefinder/WordIndexer.java
@@ -17,8 +17,6 @@
 
 package org.apache.opennlp.namefinder;
 
-import opennlp.tools.util.StringUtil;
-
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -28,6 +26,8 @@
 import java.util.Map;
 import java.util.regex.Pattern;
 
+import opennlp.tools.util.StringUtil;
+
 public class WordIndexer {
 
   private final Map<Character, Integer> char2idx;
@@ -35,11 +35,9 @@
 
   public static String UNK = "$UNK$";
   public static String NUM = "$NUM$";
-  public static String NONE = "O";
 
-  //private boolean useChars = true;
-  private boolean lowerCase = true;
-  private boolean allowUnk = true;
+  private boolean lowerCase = false;
+  private boolean allowUnk = false;
 
   private Pattern digitPattern = Pattern.compile("\\d+(,\\d+)*(\\.\\d+)?");
 
@@ -114,8 +112,9 @@
     if (lowerCase) {
       word = StringUtil.toLowerCase(word);
     }
-    if (digitPattern.matcher(word).find())
-      word = NUM;
+
+    // if (digitPattern.matcher(word).find())
+    //  word = NUM;
 
     // 2. get id of word
     Integer wordId;
@@ -140,7 +139,6 @@
     return tokenIds;
   }
 
-
   public class Ids {
 
     private int[] chars;
@@ -162,5 +160,4 @@
       this.word = word;
     }
   }
-
 }