| /* |
| * Copyright 2009-2012 by The Regents of the University of California |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * you may obtain a copy of the License from |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package edu.uci.ics.asterix.lexergenerator; |
| |
| import java.io.BufferedReader; |
| import java.io.FileNotFoundException; |
| import java.io.FileReader; |
| import java.io.Reader; |
| import java.io.InputStream; |
| import java.io.InputStreamReader; |
| import java.io.File; |
| import java.io.FileWriter; |
| import java.io.IOException; |
| import java.util.HashMap; |
| import java.util.LinkedHashMap; |
| import java.util.Map.Entry; |
| import java.util.Set; |
| import org.apache.maven.plugin.logging.Log; |
| |
| public class LexerGenerator { |
| private LinkedHashMap<String, Token> tokens = new LinkedHashMap<String, Token>(); |
| private Log logger; |
| |
| public LexerGenerator() { |
| } |
| |
| public LexerGenerator(Log logger) { |
| this.logger = logger; |
| } |
| |
| private void log(String info) { |
| if (logger == null) { |
| System.out.println(info); |
| } else { |
| logger.info(info); |
| } |
| } |
| |
| public void addToken(String rule) throws Exception { |
| Token newToken; |
| if (rule.charAt(0) == '@') { |
| newToken = new TokenAux(rule, tokens); |
| } else { |
| newToken = new Token(rule, tokens); |
| } |
| Token existingToken = tokens.get(newToken.getName()); |
| if (existingToken == null) { |
| tokens.put(newToken.getName(), newToken); |
| } else { |
| existingToken.merge(newToken); |
| } |
| } |
| |
| public void generateLexer(HashMap<String, String> config) throws Exception { |
| LexerNode main = this.compile(); |
| config.put("TOKENS_CONSTANTS", this.tokensConstants()); |
| config.put("TOKENS_IMAGES", this.tokensImages()); |
| config.put("LEXER_LOGIC", main.toJava()); |
| config.put("LEXER_AUXFUNCTIONS", replaceParams(this.auxiliaryFunctions(main), config)); |
| String[] files = { "/Lexer.java", "/LexerException.java" }; |
| String outputDir = config.get("OUTPUT_DIR"); |
| (new File(outputDir)).mkdirs(); |
| for (String file : files) { |
| String input = readFile(LexerGenerator.class.getResourceAsStream(file)); |
| String fileOut = file.replace("Lexer", config.get("LEXER_NAME")); |
| String output = replaceParams(input, config); |
| log("Generating: " + file + "\t>>\t" + fileOut); |
| FileWriter out = new FileWriter((new File(outputDir, fileOut)).toString()); |
| out.write(output); |
| out.close(); |
| log(" [done]\n"); |
| } |
| } |
| |
| public String printParsedGrammar() { |
| StringBuilder result = new StringBuilder(); |
| for (Token token : tokens.values()) { |
| result.append(token.toString()).append("\n"); |
| } |
| return result.toString(); |
| } |
| |
| private LexerNode compile() throws Exception { |
| LexerNode main = new LexerNode(); |
| for (Token token : tokens.values()) { |
| if (token instanceof TokenAux) |
| continue; |
| main.merge(token.getNode()); |
| } |
| return main; |
| } |
| |
| private String tokensImages() { |
| StringBuilder result = new StringBuilder(); |
| Set<String> uniqueTokens = tokens.keySet(); |
| for (String token : uniqueTokens) { |
| result.append(", \"<").append(token).append(">\" "); |
| } |
| return result.toString(); |
| } |
| |
| private String tokensConstants() { |
| StringBuilder result = new StringBuilder(); |
| Set<String> uniqueTokens = tokens.keySet(); |
| int i = 2; |
| for (String token : uniqueTokens) { |
| result.append(", TOKEN_").append(token).append("=").append(i).append(" "); |
| i++; |
| } |
| return result.toString(); |
| } |
| |
| private String auxiliaryFunctions(LexerNode main) { |
| StringBuilder result = new StringBuilder(); |
| Set<String> functions = main.neededAuxFunctions(); |
| for (String token : functions) { |
| result.append("private int parse_" + token |
| + "(char currentChar) throws IOException, [LEXER_NAME]Exception{\n"); |
| result.append(tokens.get(token).getNode().toJavaAuxFunction()); |
| result.append("\n}\n\n"); |
| } |
| return result.toString(); |
| } |
| |
| private static String readFile(Reader input) throws FileNotFoundException, IOException { |
| StringBuffer fileData = new StringBuffer(1000); |
| BufferedReader reader = new BufferedReader(input); |
| char[] buf = new char[1024]; |
| int numRead = 0; |
| while ((numRead = reader.read(buf)) != -1) { |
| String readData = String.valueOf(buf, 0, numRead); |
| fileData.append(readData); |
| buf = new char[1024]; |
| } |
| reader.close(); |
| return fileData.toString(); |
| } |
| |
| private static String readFile(InputStream input) throws FileNotFoundException, IOException { |
| if (input == null) { |
| throw new FileNotFoundException(); |
| } |
| return readFile(new InputStreamReader(input)); |
| } |
| |
| private static String readFile(String fileName) throws FileNotFoundException, IOException { |
| return readFile(new FileReader(fileName)); |
| } |
| |
| private static String replaceParams(String input, HashMap<String, String> config) { |
| for (Entry<String, String> param : config.entrySet()) { |
| String key = "\\[" + param.getKey() + "\\]"; |
| String value = param.getValue(); |
| input = input.replaceAll(key, value); |
| } |
| return input; |
| } |
| |
| public static void main(String args[]) throws Exception { |
| if (args.length == 0 || args[0] == "--help" || args[0] == "-h") { |
| System.out.println("LexerGenerator\nusage: java LexerGenerator <configuration file>"); |
| return; |
| } |
| |
| LexerGenerator lexer = new LexerGenerator(); |
| HashMap<String, String> config = new HashMap<String, String>(); |
| |
| System.out.println("Config file:\t" + args[0]); |
| String input = readFile(args[0]); |
| boolean tokens = false; |
| for (String line : input.split("\r?\n")) { |
| line = line.trim(); |
| if (line.length() == 0 || line.charAt(0) == '#') |
| continue; |
| if (tokens == false && !line.equals("TOKENS:")) { |
| config.put(line.split("\\s*:\\s*")[0], line.split("\\s*:\\s*")[1]); |
| } else if (line.equals("TOKENS:")) { |
| tokens = true; |
| } else { |
| lexer.addToken(line); |
| } |
| } |
| |
| String parsedGrammar = lexer.printParsedGrammar(); |
| lexer.generateLexer(config); |
| System.out.println("\nGenerated grammar:"); |
| System.out.println(parsedGrammar); |
| } |
| |
| } |