| # ************************************************************* | |
| # | |
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # | |
| # ************************************************************* | |
| import sys, os.path | |
| from globals import * | |
| import macroparser | |
| class EOF(Exception): | |
| def __init__ (self): | |
| pass | |
| def str (self): | |
| return "end of file" | |
| class BOF(Exception): | |
| def __init__ (self): | |
| pass | |
| def str (self): | |
| return "beginning of file" | |
| def removeHeaderQuotes (orig): | |
| if len(orig) <= 2: | |
| return orig | |
| elif orig[0] == orig[-1] == '"': | |
| return orig[1:-1] | |
| elif orig[0] == '<' and orig[-1] == '>': | |
| return orig[1:-1] | |
| return orig | |
| def dumpTokens (tokens, toError=False): | |
| scope = 0 | |
| indent = " " | |
| line = '' | |
| chars = '' | |
| for token in tokens: | |
| if token in '{<': | |
| if len(line) > 0: | |
| chars += indent*scope + line + "\n" | |
| line = '' | |
| chars += indent*scope + token + "\n" | |
| scope += 1 | |
| elif token in '}>': | |
| if len(line) > 0: | |
| chars += indent*scope + line + "\n" | |
| line = '' | |
| scope -= 1 | |
| chars += indent*scope + token | |
| elif token == ';': | |
| if len(line) > 0: | |
| chars += indent*scope + line + ";\n" | |
| line = '' | |
| else: | |
| chars += ";\n" | |
| elif len(token) > 0: | |
| line += token + ' ' | |
| if len(line) > 0: | |
| chars += line | |
| chars += "\n" | |
| if toError: | |
| sys.stderr.write(chars) | |
| else: | |
| sys.stdout.write(chars) | |
| class HeaderData(object): | |
| def __init__ (self): | |
| self.defines = {} | |
| self.tokens = [] | |
| class SrcLexer(object): | |
| """Lexicographical analyzer for .src format. | |
| The role of a lexer is to parse the source file and break it into | |
| appropriate tokens. Such tokens are later passed to a parser to | |
| build the syntax tree. | |
| """ | |
| headerCache = {} | |
| VISIBLE = 0 | |
| INVISIBLE_PRE = 1 | |
| INVISIBLE_POST = 2 | |
| def __init__ (self, chars, filepath = None): | |
| self.filepath = filepath | |
| self.parentLexer = None | |
| self.chars = chars | |
| self.bufsize = len(self.chars) | |
| # TODO: use parameters for this | |
| # Properties that can be copied. | |
| self.headerDict = dict () | |
| self.debug = False | |
| self.debugMacro = False | |
| self.includeDirs = list () | |
| self.expandHeaders = True | |
| self.inMacroDefine = False | |
| self.stopOnHeader = False | |
| def copyProperties (self, other): | |
| """Copy properties from another instance of SrcLexer.""" | |
| # TODO: use parameters for this | |
| self.headerDict = other.headerDict | |
| self.debug = other.debug | |
| self.debugMacro = other.debugMacro | |
| self.includeDirs = other.includeDirs[:] | |
| self.expandHeaders = other.expandHeaders | |
| self.inMacroDefine = other.inMacroDefine | |
| self.stopOnHeader = other.stopOnHeader | |
| def init (self): | |
| self.firstNonBlank = '' | |
| self.token = '' | |
| self.tokens = [] | |
| self.defines = {} | |
| self.visibilityStack = [] | |
| def getTokens (self): | |
| return self.tokens | |
| def getDefines (self): | |
| return self.defines | |
| def nextPos (self, i): | |
| while True: | |
| i += 1 | |
| try: | |
| c = self.chars[i] | |
| except IndexError: | |
| raise EOF | |
| if ord(c) in [0x0D]: | |
| continue | |
| break | |
| return i | |
| def prevPos (self, i): | |
| while True: | |
| i -= 1 | |
| try: | |
| c = self.chars[i] | |
| except IndexError: | |
| raise BOF | |
| if ord(c) in [0x0D]: | |
| continue | |
| break | |
| return i | |
| def isCodeVisible (self): | |
| if len(self.visibilityStack) == 0: | |
| return True | |
| for item in self.visibilityStack: | |
| if item != SrcLexer.VISIBLE: | |
| return False | |
| return True | |
| def tokenize (self): | |
| self.init() | |
| i = 0 | |
| while True: | |
| c = self.chars[i] | |
| if self.firstNonBlank == '' and not c in [' ', "\n", "\t"]: | |
| # Store the first non-blank in a line. | |
| self.firstNonBlank = c | |
| elif c == "\n": | |
| self.firstNonBlank = '' | |
| if c == '#': | |
| i = self.pound(i) | |
| elif c == '/': | |
| i = self.slash(i) | |
| elif c == "\n": | |
| i = self.lineBreak(i) | |
| elif c == '"': | |
| i = self.doubleQuote(i) | |
| elif c in [' ', "\t"]: | |
| i = self.blank(i) | |
| elif c in ";()[]{}<>,=+-*": | |
| # Any outstanding single-character token. | |
| i = self.anyToken(i, c) | |
| elif self.isCodeVisible(): | |
| self.token += c | |
| try: | |
| i = self.nextPos(i) | |
| except EOF: | |
| break | |
| if len(self.token): | |
| self.tokens.append(self.token) | |
| if not self.parentLexer and self.debug: | |
| progress ("-"*68 + "\n") | |
| progress ("All defines found in this translation unit:\n") | |
| keys = sorted(self.defines.keys()) | |
| for key in keys: | |
| progress ("@ %s\n"%key) | |
| def dumpTokens (self, toError=False): | |
| dumpTokens(self.tokens, toError) | |
| def maybeAddToken (self): | |
| if len(self.token) > 0: | |
| self.tokens.append(self.token) | |
| self.token = '' | |
| #-------------------------------------------------------------------- | |
| # character handlers | |
| def blank (self, i): | |
| if not self.isCodeVisible(): | |
| return i | |
| self.maybeAddToken() | |
| return i | |
| def pound (self, i): | |
| if self.inMacroDefine: | |
| return i | |
| if not self.firstNonBlank == '#': | |
| return i | |
| self.maybeAddToken() | |
| # We are in preprocessing mode. | |
| # Get the macro command name '#<command> .....' | |
| command, define, buf = '', '', '' | |
| firstNonBlank = False | |
| while True: | |
| try: | |
| i = self.nextPos(i) | |
| c = self.chars[i] | |
| if c == '\\' and self.chars[self.nextPos(i)] == "\n": | |
| i = self.nextPos(i) | |
| continue | |
| except EOF: | |
| break | |
| if c == "\n": | |
| if len(buf) > 0 and len(command) == 0: | |
| command = buf | |
| i = self.prevPos(i) | |
| break | |
| elif c in [' ', "\t"]: | |
| if not firstNonBlank: | |
| # Ignore any leading blanks after the '#'. | |
| continue | |
| if len(command) == 0: | |
| command = buf | |
| buf = '' | |
| else: | |
| buf += ' ' | |
| elif c == '(': | |
| if len(buf) > 0 and len(command) == 0: | |
| command = buf | |
| buf += c | |
| else: | |
| if not firstNonBlank: | |
| firstNonBlank = True | |
| buf += c | |
| if command == 'define': | |
| self.handleMacroDefine(buf) | |
| elif command == 'include': | |
| self.handleMacroInclude(buf) | |
| elif command == 'ifdef': | |
| defineName = buf.strip() | |
| if defineName in self.defines: | |
| self.visibilityStack.append(SrcLexer.VISIBLE) | |
| else: | |
| self.visibilityStack.append(SrcLexer.INVISIBLE_PRE) | |
| elif command == 'ifndef': | |
| defineName = buf.strip() | |
| if defineName in self.defines: | |
| self.visibilityStack.append(SrcLexer.INVISIBLE_PRE) | |
| else: | |
| self.visibilityStack.append(SrcLexer.VISIBLE) | |
| elif command == 'if': | |
| if self.evalCodeVisibility(buf): | |
| self.visibilityStack.append(SrcLexer.VISIBLE) | |
| else: | |
| self.visibilityStack.append(SrcLexer.INVISIBLE_PRE) | |
| elif command == 'elif': | |
| if len(self.visibilityStack) == 0: | |
| raise ParseError ('') | |
| if self.visibilityStack[-1] == SrcLexer.VISIBLE: | |
| self.visibilityStack[-1] = SrcLexer.INVISIBLE_POST | |
| elif self.visibilityStack[-1] == SrcLexer.INVISIBLE_PRE: | |
| # Evaluate only if the current visibility is false. | |
| if self.evalCodeVisibility(buf): | |
| self.visibilityStack[-1] = SrcLexer.VISIBLE | |
| elif command == 'else': | |
| if len(self.visibilityStack) == 0: | |
| raise ParseError ('') | |
| if self.visibilityStack[-1] == SrcLexer.VISIBLE: | |
| self.visibilityStack[-1] = SrcLexer.INVISIBLE_POST | |
| if self.visibilityStack[-1] == SrcLexer.INVISIBLE_PRE: | |
| self.visibilityStack[-1] = SrcLexer.VISIBLE | |
| elif command == 'endif': | |
| if len(self.visibilityStack) == 0: | |
| raise ParseError ('') | |
| self.visibilityStack.pop() | |
| elif command == 'undef': | |
| pass | |
| elif command in ['error', 'pragma']: | |
| pass | |
| else: | |
| print("'%s' '%s'"%(command, buf)) | |
| print(self.filepath) | |
| sys.exit(0) | |
| return i | |
| def evalCodeVisibility (self, buf): | |
| try: | |
| return eval(buf) | |
| except: | |
| return True | |
| def handleMacroDefine (self, buf): | |
| mparser = macroparser.MacroParser(buf) | |
| mparser.debug = self.debugMacro | |
| mparser.parse() | |
| macro = mparser.getMacro() | |
| if macro: | |
| self.defines[macro.name] = macro | |
| def handleMacroInclude (self, buf): | |
| # Strip excess string if any. | |
| pos = buf.find(' ') | |
| if pos >= 0: | |
| buf = buf[:pos] | |
| headerSub = removeHeaderQuotes(buf) | |
| if not self.expandHeaders: | |
| # We don't want to expand headers. Bail out. | |
| if self.debug: | |
| progress ("%s ignored\n"%headerSub) | |
| return | |
| defines = {} | |
| headerPath = None | |
| for includeDir in self.includeDirs: | |
| hpath = includeDir + '/' + headerSub | |
| if os.path.isfile(hpath) and hpath != self.filepath: | |
| headerPath = hpath | |
| break | |
| if not headerPath: | |
| error("included header file " + headerSub + " not found\n", self.stopOnHeader) | |
| return | |
| if self.debug: | |
| progress ("%s found\n"%headerPath) | |
| if headerPath in self.headerDict: | |
| if self.debug: | |
| progress ("%s already included\n"%headerPath) | |
| return | |
| if headerPath in SrcLexer.headerCache: | |
| if self.debug: | |
| progress ("%s in cache\n"%headerPath) | |
| for key in list(SrcLexer.headerCache[headerPath].defines.keys()): | |
| self.defines[key] = SrcLexer.headerCache[headerPath].defines[key] | |
| return | |
| chars = open(headerPath, 'r').read() | |
| mclexer = SrcLexer(chars, headerPath) | |
| mclexer.copyProperties(self) | |
| mclexer.parentLexer = self | |
| mclexer.tokenize() | |
| hdrData = HeaderData() | |
| hdrData.tokens = mclexer.getTokens() | |
| headerDefines = mclexer.getDefines() | |
| for key in list(headerDefines.keys()): | |
| defines[key] = headerDefines[key] | |
| hdrData.defines[key] = headerDefines[key] | |
| self.headerDict[headerPath] = True | |
| SrcLexer.headerCache[headerPath] = hdrData | |
| # Update the list of headers that have already been expanded. | |
| for key in list(mclexer.headerDict.keys()): | |
| self.headerDict[key] = True | |
| if self.debug: | |
| progress ("defines found in header %s:\n"%headerSub) | |
| for key in list(defines.keys()): | |
| progress (" '%s'\n"%key) | |
| for key in list(defines.keys()): | |
| self.defines[key] = defines[key] | |
| def slash (self, i): | |
| if not self.isCodeVisible(): | |
| return i | |
| if i < self.bufsize - 1 and self.chars[i+1] == '/': | |
| # Parse line comment. | |
| line = '' | |
| i += 2 | |
| while i < self.bufsize: | |
| c = self.chars[i] | |
| if ord(c) in [0x0A, 0x0D]: | |
| return i - 1 | |
| line += c | |
| i += 1 | |
| self.token = '' | |
| elif i < self.bufsize - 1 and self.chars[i+1] == '*': | |
| comment = '' | |
| i += 2 | |
| while i < self.bufsize: | |
| c = self.chars[i] | |
| if c == '/' and self.chars[i-1] == '*': | |
| return i | |
| comment += c | |
| i += 1 | |
| else: | |
| return self.anyToken(i, '/') | |
| return i | |
| def lineBreak (self, i): | |
| if not self.isCodeVisible(): | |
| return i | |
| self.maybeAddToken() | |
| return i | |
| def doubleQuote (self, i): | |
| if not self.isCodeVisible(): | |
| return i | |
| literal = '' | |
| i += 1 | |
| while i < self.bufsize: | |
| c = self.chars[i] | |
| if c == '"': | |
| self.tokens.append('"'+literal+'"') | |
| break | |
| literal += c | |
| i += 1 | |
| return i | |
| def anyToken (self, i, token): | |
| if not self.isCodeVisible(): | |
| return i | |
| self.maybeAddToken() | |
| self.token = token | |
| self.maybeAddToken() | |
| return i |