commit 0e96b9cd8c1267acee02684bcac5be89d60b3bf4
author asubbu90 <asubbu.90@gmail.com> Fri Dec 01 22:31:29 2023 +0530
committer GitHub <noreply@github.com> Fri Dec 01 12:01:29 2023 -0500
tree 0d01efcbf1fe8cb77179a52f5a9143281a790f1e
parent ad0c9cca9792a6525d4955f5be8cf787bb9f7774

TruncateTokenFilterFactory now accepts lengths > 127 (#12507)

lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterFactory.java

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterFactory.java
index 3c508b9..ba57a67 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterFactory.java
@@ -46,11 +46,11 @@
   public static final String NAME = "truncate";
 
   public static final String PREFIX_LENGTH_KEY = "prefixLength";
-  private final byte prefixLength;
+  private final int prefixLength;
 
   public TruncateTokenFilterFactory(Map<String, String> args) {
     super(args);
-    prefixLength = Byte.parseByte(get(args, PREFIX_LENGTH_KEY, "5"));
+    prefixLength = Integer.parseInt(get(args, PREFIX_LENGTH_KEY, "5"));
     if (prefixLength < 1)
       throw new IllegalArgumentException(
           PREFIX_LENGTH_KEY + " parameter must be a positive number: " + prefixLength);

lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilterFactory.java

diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilterFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilterFactory.java
index 2fed6c2..f537b93 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilterFactory.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilterFactory.java
@@ -68,4 +68,23 @@
                 TruncateTokenFilterFactory.PREFIX_LENGTH_KEY
                     + " parameter must be a positive number: -5"));
   }
+
+  /** Test that takes length greater than byte limit accepts it */
+  public void testLengthGreaterThanByteLimitArgument() throws Exception {
+    Reader reader =
+        new StringReader(
+            "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvw128characters From here");
+    TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+    ((Tokenizer) stream).setReader(reader);
+    stream =
+        tokenFilterFactory("Truncate", TruncateTokenFilterFactory.PREFIX_LENGTH_KEY, "128")
+            .create(stream);
+    assertTokenStreamContents(
+        stream,
+        new String[] {
+          "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvw1",
+          "From",
+          "here"
+        });
+  }
 }