Use STX and ETX for start end seq chars in normalizer
diff --git a/tf-ner-poc/src/main/java/org/apache/opennlp/normalizer/Normalizer.java b/tf-ner-poc/src/main/java/org/apache/opennlp/normalizer/Normalizer.java
index 5629a06..f0261fe 100644
--- a/tf-ner-poc/src/main/java/org/apache/opennlp/normalizer/Normalizer.java
+++ b/tf-ner-poc/src/main/java/org/apache/opennlp/normalizer/Normalizer.java
@@ -39,7 +39,7 @@
public class Normalizer {
- private static final char END_MARKER = 'E';
+ private static final char END_MARKER = 3;
private final Session session;
private final Map<Character, Integer> sourceCharMap;
diff --git a/tf-ner-poc/src/main/python/normalizer/normalizer.py b/tf-ner-poc/src/main/python/normalizer/normalizer.py
index e720cc1..6be491c 100644
--- a/tf-ner-poc/src/main/python/normalizer/normalizer.py
+++ b/tf-ner-poc/src/main/python/normalizer/normalizer.py
@@ -211,9 +211,9 @@
target_char_dict = encode_chars(target_train + target_dev + target_test)
- # TODO: Find better chars for begin and end markers
- target_char_dict['S'] = len(target_char_dict)
- target_char_dict['E'] = len(target_char_dict)
+ # char id 2 is STX (Start of Text), and 3 ETX (End of Text)
+ target_char_dict[chr(2)] = len(target_char_dict)
+ target_char_dict[chr(3)] = len(target_char_dict)
target_dict_rev = {v: k for k, v in target_char_dict.items()}
