| /* $Id$ */ |
| |
| /** |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.manifoldcf.scriptengine; |
| |
| import java.io.*; |
| |
| /** Convert a sequence of characters into a stream of tokens. |
| */ |
| public class BasicTokenStream implements TokenStream |
| { |
| protected CharacterStream characterStream; |
| protected Token currentToken = null; |
| |
| public BasicTokenStream(Reader reader) |
| { |
| this.characterStream = new CharacterStream(reader); |
| } |
| |
| public Token peek() |
| throws ScriptException |
| { |
| if (currentToken == null) |
| currentToken = parseNextToken(); |
| return currentToken; |
| } |
| |
| public void skip() |
| { |
| currentToken = null; |
| } |
| |
| // Protected methods |
| |
| protected Token parseNextToken() |
| throws ScriptException |
| { |
| char y; |
| int lineNumber; |
| int characterNumber; |
| |
| // Skip to the start of the next token (or exit if no token start can be found) |
| while (true) |
| { |
| |
| // Skip white space |
| while (true) |
| { |
| // Remember current position so we can properly characterize the token. |
| lineNumber = characterStream.getLineNumber(); |
| characterNumber = characterStream.getCharacterNumber(); |
| int x = characterStream.peek(); |
| if (x == -1) |
| return null; |
| y = (char)x; |
| if (y > ' ') |
| break; |
| characterStream.skip(); |
| } |
| |
| // Is it a comment? |
| if (y == '#') |
| { |
| // Skip to the end of the line, or the end of the text. |
| while (true) |
| { |
| int x = characterStream.peek(); |
| if (x == -1) |
| return null; |
| characterStream.skip(); |
| y = (char)x; |
| if (y == '\n') |
| break; |
| } |
| continue; |
| } |
| break; |
| } |
| |
| // At the start of the next token. |
| if (isQuoteCharacter(y)) |
| { |
| // Legal token character |
| StringBuilder tokenBuffer = new StringBuilder(); |
| char quoteMark = y; |
| characterStream.skip(); |
| while (true) |
| { |
| int x = characterStream.peek(); |
| if (x == -1) |
| break; |
| y = (char)x; |
| characterStream.skip(); |
| if (y == quoteMark) |
| // End of the quotation |
| break; |
| if (y == '\\') |
| { |
| // Escape character |
| x = characterStream.peek(); |
| if (x != -1) |
| { |
| tokenBuffer.append((char)x); |
| characterStream.skip(); |
| } |
| } |
| else |
| tokenBuffer.append(y); |
| } |
| return new Token(Token.TOKEN_STRING,tokenBuffer.toString(),lineNumber,characterNumber); |
| } |
| else if (isTokenCharacter(y)) |
| { |
| // Legal token character |
| StringBuilder tokenBuffer = new StringBuilder(); |
| while (true) |
| { |
| int x = characterStream.peek(); |
| if (x == -1) |
| break; |
| y = (char)x; |
| if (isTokenCharacter(y) || isNumberCharacter(y)) |
| { |
| // Tokens can include numbers, just so long they aren't the first character |
| characterStream.skip(); |
| tokenBuffer.append(y); |
| } |
| else |
| break; |
| } |
| return new Token(Token.TOKEN_TOKEN,tokenBuffer.toString(),lineNumber,characterNumber); |
| } |
| else if (isNumberCharacter(y)) |
| { |
| StringBuilder tokenBuffer = new StringBuilder(); |
| boolean seenDot = false; |
| while (true) |
| { |
| int x = characterStream.peek(); |
| if (x == -1) |
| break; |
| y = (char)x; |
| if (isNumberCharacter(y)) |
| { |
| tokenBuffer.append(y); |
| characterStream.skip(); |
| } |
| else if (y == '.' && seenDot == false) |
| { |
| tokenBuffer.append(y); |
| seenDot = true; |
| characterStream.skip(); |
| } |
| else |
| break; |
| } |
| if (seenDot) |
| return new Token(Token.TOKEN_FLOAT,tokenBuffer.toString(),lineNumber,characterNumber); |
| else |
| return new Token(Token.TOKEN_INTEGER,tokenBuffer.toString(),lineNumber,characterNumber); |
| } |
| else |
| { |
| // Set a punctuation token |
| characterStream.skip(); |
| if (y == '=' || y == '!' || y == '>' || y == '<' || y == '&' || y == '|') |
| { |
| int x = characterStream.peek(); |
| if (x != -1) |
| { |
| char q = (char)x; |
| if (y == '=' && q == '=' || |
| y == '!' && q == '=' || |
| y == '>' && q == '=' || |
| y == '<' && q == '=' || |
| y == '&' && q == '&' || |
| y == '|' && q == '|' || |
| y == '>' && q == '>' || |
| y == '<' && q == '<') |
| { |
| characterStream.skip(); |
| return new Token(Token.TOKEN_PUNCTUATION,new StringBuilder().append(y).append(q).toString(), |
| lineNumber,characterNumber); |
| } |
| } |
| } |
| return new Token(Token.TOKEN_PUNCTUATION,new StringBuilder().append(y).toString(), |
| lineNumber,characterNumber); |
| } |
| } |
| |
| protected static boolean isQuoteCharacter(char x) |
| { |
| return (x == '\'' || x == '"'); |
| } |
| |
| protected static boolean isNumberCharacter(char x) |
| { |
| return (x >= '0' && x <= '9'); |
| } |
| |
| protected static boolean isTokenCharacter(char x) |
| { |
| if (x >= 'a' && x <= 'z') |
| return true; |
| if (x >= 'A' && x <='Z') |
| return true; |
| if (x == '_' || x == '$' || x == '@') |
| return true; |
| return false; |
| } |
| |
| } |