commit 52409ead93742bbf36a51b00a2148002161b5b2d
author Laszlo Kishalmi <laszlo.kishalmi@gmail.com> Wed Aug 31 21:28:00 2022 -0700
committer Laszlo Kishalmi <laszlo.kishalmi@gmail.com> Sat Sep 03 12:07:48 2022 -0700
tree 6460303996e02412bea38add749c2940451f979e
parent 95f381fadc5d046ef7ab416617d95620ea0efa08

A slightly better CharStream implementation

ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java

diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java
index e830b63..5f359d8 100644
--- a/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java
+++ b/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java
@@ -28,8 +28,8 @@
  */
 public class LexerInputCharStream implements CharStream {
     private final LexerInput input;
-    private final StringBuilder readBuffer = new StringBuilder();
 
+    private int tokenMark = Integer.MAX_VALUE;
     private int index = 0;
 
     public LexerInputCharStream(LexerInput input) {
@@ -38,8 +38,12 @@
 
     @Override
     public String getText(Interval intrvl) {
-        int end = Math.min(intrvl.b + 1, readBuffer.length());
-        return readBuffer.substring(intrvl.a, end);
+        if (intrvl.a < tokenMark) {
+            throw new UnsupportedOperationException("Read before the current token start is not supported: " + intrvl.a + " < " + tokenMark);
+        }
+        int start = intrvl.a - tokenMark;
+        int end = intrvl.b - tokenMark + 1;
+        return String.valueOf(input.readText(start, end));
     }
 
     @Override
@@ -97,9 +101,6 @@
 
     private int read() {
         int ret = input.read();
-        if ((readBuffer.length() == index) && (ret != EOF)) {
-            readBuffer.append((char)ret);
-        }
         index += 1;
         return ret;
     }
@@ -109,6 +110,10 @@
         input.backup(count);
     }
 
+    public final void markToken() {
+        tokenMark = index;
+    }
+
     @Override
     public int size() {
         throw new UnsupportedOperationException("Stream size is unknown.");

ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java

diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java
index a30199c..dcbced4 100644
--- a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java
+++ b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java
@@ -36,10 +36,12 @@
 
     private final TokenFactory<TomlTokenId> tokenFactory;
     private final org.tomlj.internal.TomlLexer lexer;
+    private final LexerInputCharStream input;
 
     public TomlLexer(LexerRestartInfo<TomlTokenId> info) {
         this.tokenFactory = info.tokenFactory();
-        this.lexer = new org.tomlj.internal.TomlLexer(new LexerInputCharStream(info.input()));
+        this.input = new LexerInputCharStream(info.input());
+        this.lexer = new org.tomlj.internal.TomlLexer(input);
         if (info.state() != null) {
             ((LexerState) info.state()).restore(lexer);
         }
@@ -130,6 +132,7 @@
     }
 
     private Token<TomlTokenId> token(TomlTokenId id) {
+        input.markToken();
         return tokenFactory.createToken(id);
     }