| /+ |
| * Copyright (c) 1999-2006 by Digital Mars |
| * All Rights Reserved |
| * written by Walter Bright www.digitalmars.com |
| * License for redistribution is by either the Artistic License in artistic.txt, or the GNU General Public License in gnu.txt. |
| * See the included readme.txt for details. |
| * D Language conversion by: J Duncan |
| +/ |
| |
| /** |
| * d language lexer |
| */ |
| |
| module dparser.Lexer; |
| |
| import dparser.Root; |
| |
| import dparser.Tokens; |
| import dparser.Token; |
| import dparser.Keyword; |
| |
| import dparser.Types; |
| |
| import dparser.Module; |
| import dparser.Identifier; |
| import dparser.unialpha; |
| |
| import dparser.OutBuffer; |
| |
| //private import std.ctype; |
| //private import std.string; |
| //import dwf.core.debugapi; |
| |
| int errno = 0; |
| |
| //#if _WIN32 && __DMC__ |
| // from \dm\src\include\setlocal.h |
| //extern "C" char * __cdecl __locale_decpoint; |
| char* __locale_decpoint; |
| //#endif |
| //const uint LS = 0x2028; // UTF line separator |
| //const uint PS = 0x2029; // UTF paragraph separator |
| |
| //extern int isUniAlpha(unsigned u); |
| //extern int HtmlNamedEntity(unsigned char *p, int length); |
| |
| /** |
| * Lexer object |
| */ |
| |
| class Lexer |
| { |
| static Identifier[char[]] stringtable; |
| static OutBuffer stringbuffer; |
| static Token * freelist; |
| |
| Token token; // current token |
| Module mod; // current module |
| Loc loc; // for error messages |
| ubyte *base; // pointer to start of buffer |
| ubyte *end; // past end of buffer |
| ubyte *p; // current character |
| int doDocComment; // collect doc comment information |
| int anyToken; // !=0 means seen at least one token |
| int commentToken; // !=0 means comments are TOKcomment's |
| |
| |
| this(Module mod, ubyte* base, uint begoffset, uint endoffset, int doDocComment, int commentToken) |
| { |
| if (stringbuffer is null) { |
| stringbuffer = new OutBuffer; |
| } |
| loc = Loc(mod, 1); |
| |
| this.base = base; |
| this.end = base + endoffset; |
| this.p = base + begoffset; |
| this.mod = mod; |
| this.doDocComment = doDocComment; |
| this.commentToken = commentToken; |
| |
| /* |
| * If first line starts with '#!', ignore the line |
| */ |
| |
| if (p[0] == '#' && p[1] == '!') { |
| p += 2; |
| while (true) { |
| ubyte c = *p; |
| switch (c) { |
| case '\n': |
| p++; |
| break; |
| |
| case '\r': |
| p++; |
| if (*p == '\n') { |
| p++; |
| } |
| break; |
| |
| case 0: |
| case 0x1A: |
| break; |
| |
| default: |
| if (c & 0x80) { |
| uint u = decodeUTF(); |
| if (u == PS || u == LS) { |
| break; |
| } |
| } |
| p++; |
| continue; |
| } |
| break; |
| } |
| |
| loc.linnum = 2; |
| } |
| } |
| |
| |
| |
| // generate a unique identifier for this string |
| static Identifier idPool(in char[] str) { |
| // StringValue sv; |
| // uint len = s.length; |
| // StringValue sv = stringtable.update(s, len); |
| // Identifier* id = cast(Identifier*) sv.ptrvalue; |
| // if( id is null ) |
| if ((str in stringtable) == null) { |
| stringtable[str] = new Identifier(str, TOK.TOKidentifier); |
| } |
| return(stringtable[str]); |
| } |
| |
| static void initKeywords() { |
| // build character map |
| cmtable_init(); |
| |
| // create keyword tokens & identifiers |
| dparser.Keyword.initKeywords(); |
| |
| // create standard lexer tokens |
| dparser.Token.createLexerTokens(); |
| } |
| |
| // Combine two document comments into one. |
| static char[] combineComments(char[] c1, char[] c2) { |
| char[] c = c2; |
| if (c1.length) { |
| c = c1; |
| if (c2.length) { |
| c = c1 ~ "\n" ~ c2; |
| } |
| } |
| return(c); |
| } |
| |
| // Decode UTF character. Issue error messages for invalid sequences. Return decoded character, advance p to last character in UTF sequence. |
| //! fix |
| uint decodeUTF() { |
| ubyte * s = p; |
| ubyte c = *s; |
| |
| assert(c & 0x80); |
| if (!(c & 0x80)) { |
| return(c); |
| } |
| |
| return(cast(uint)'X'); |
| /* |
| * dchar u; |
| * uint len; |
| * |
| * |
| * |
| * // Check length of remaining string up to 6 UTF-8 characters |
| * for( len = 1; len < 6 && s[len]; len++ ) |
| * { |
| * |
| * } |
| * /+ |
| * uint idx = 0; |
| * char* msg = utf_decodeChar( s, len, &idx, &u ); |
| * p += idx - 1; |
| * if( msg ) |
| * { |
| * error(msg); |
| * } |
| * +/ |
| * return u; |
| */ |
| } |
| |
| void error(...) { |
| if ((mod !is null) && !global.gag) { |
| writefln(formatLoc(loc, _arguments, _argptr)); |
| /* |
| * char[] p = loc.toChars(); |
| * if( p.length ) |
| * writef( "%s: ", p ); |
| * writefx( stdout, _arguments, _argptr, 1 ); |
| */ |
| if (global.errors >= global.max_errors) { // moderate blizzard of cascading messages |
| throw new Exception("too many errors"); |
| } |
| } |
| |
| global.errors++; |
| } |
| |
| void errorLoc(Loc loc, ...) { |
| if ((mod !is null) && !global.gag) { |
| writefln(formatLoc(loc, _arguments, _argptr)); |
| /* |
| * char[] p = loc.toChars(); |
| * if( p.length ) |
| * writef("%s: ", p); |
| * writefx(stdout, _arguments, _argptr, 1); |
| */ |
| if (global.errors >= 20) { // moderate blizzard of cascading messages |
| throw new Exception("too many errors"); |
| } |
| } |
| |
| global.errors++; |
| } |
| |
| |
| TOK nextToken() { |
| if (token.next) { |
| Token* t = token.next; |
| memcpy(&token, t, Token.sizeof); |
| // t.next = freelist; |
| // freelist = t; |
| } |
| else { |
| scan(&token); |
| } |
| // token.print(); |
| return(token.value); |
| } |
| |
| Token* peek(inout Token ct) { |
| Token* t; |
| |
| if (ct.next) { |
| t = ct.next; |
| } |
| else { |
| t = new Token; |
| scan(t); |
| t.next = null; |
| ct.next = t; |
| } |
| return(t); |
| } |
| |
| // Turn next token in buffer into a token. |
| |
| void scan(Token* t) { |
| // debug writefln("scan token"); |
| uint lastLine = loc.linnum; |
| uint linnum; |
| |
| t.blockComment = null; |
| t.lineComment = null; |
| while (true) { |
| t.ptr = p; |
| // debug writefln( " p = %d, *p = ", cast(uint)p, cast(char)*p ); |
| switch (*p) { |
| case 0: |
| case 0x1a: |
| t.value = TOK.TOKeof; // end of file |
| // debug writefln( " EOF" ); |
| return; |
| |
| case ' ': |
| case '\t': |
| case '\v': |
| case '\f': |
| p++; |
| // debug writefln( " whitespace" ); |
| continue; // skip white space |
| |
| case '\r': |
| // debug writefln( " cr" ); |
| p++; |
| if (*p != '\n') { // if CR stands by itself |
| loc.linnum++; |
| } |
| continue; // skip white space |
| |
| case '\n': |
| // debug writefln( " nl" ); |
| p++; |
| loc.linnum++; |
| continue; // skip white space |
| |
| case '0': |
| case '1': |
| case '2': |
| case '3': |
| case '4': |
| case '5': |
| case '6': |
| case '7': |
| case '8': |
| case '9': |
| t.value = number(t); |
| return; |
| |
| /* |
| * #if CSTRINGS |
| * case '\'': |
| * t.value = charConstant(t, 0); |
| * return; |
| * |
| * case '"': |
| * t.value = stringConstant(t,0); |
| * return; |
| * |
| * case 'l': |
| * case 'L': |
| * if( p[1] == '\'') |
| * { |
| * p++; |
| * t.value = charConstant(t, 1); |
| * return; |
| * } |
| * else if( p[1] == '"') |
| * { |
| * p++; |
| * t.value = stringConstant(t, 1); |
| * return; |
| * } |
| * #else |
| */ |
| case '\'': |
| // debug writefln( " char" ); |
| t.value = charConstant(t, 0); |
| return; |
| |
| case 'r': |
| // debug writefln( " wysiwyg" ); |
| if (p[1] != '"') { |
| goto case_ident; |
| } |
| p++; |
| |
| case '`': |
| t.value = wysiwygStringConstant(t, *p); |
| return; |
| |
| case 'x': |
| // debug writefln( " hex string" ); |
| if (p[1] != '"') { |
| goto case_ident; |
| } |
| p++; |
| t.value = hexStringConstant(t); |
| return; |
| |
| |
| case '"': |
| // debug writefln( " string" ); |
| t.value = escapeStringConstant(t, 0); |
| // debug writefln( t.ustring ); |
| return; |
| |
| case '\\': // escaped string literal |
| // debug writefln( " escaped string literal" ); |
| uint c; |
| stringbuffer.offset = 0; |
| do { |
| p++; |
| c = escapeSequence(); |
| stringbuffer.write(c); |
| } while (*p == '\\'); |
| // t.len = stringbuffer.offset; |
| // stringbuffer.write(cast(byte)0); |
| t.ustring = stringbuffer.toString; |
| // memcpy( t.ustring.ptr, stringbuffer.data, stringbuffer.offset ); |
| t.postfix = 0; |
| t.value = TOK.TOKstring; |
| return; |
| |
| case 'l': |
| case 'L': |
| // #endif |
| |
| case 'a': |
| case 'b': |
| case 'c': |
| case 'd': |
| case 'e': |
| case 'f': |
| case 'g': |
| case 'h': |
| case 'i': |
| case 'j': |
| case 'k': |
| case 'm': |
| case 'n': |
| case 'o': |
| case 'p': |
| case 'q': /*case 'r':*/ |
| case 's': |
| case 't': |
| case 'u': |
| case 'v': |
| case 'w': /*case 'x':*/ |
| case 'y': |
| case 'z': |
| case 'A': |
| case 'B': |
| case 'C': |
| case 'D': |
| case 'E': |
| case 'F': |
| case 'G': |
| case 'H': |
| case 'I': |
| case 'J': |
| case 'K': |
| case 'M': |
| case 'N': |
| case 'O': |
| case 'P': |
| case 'Q': |
| case 'R': |
| case 'S': |
| case 'T': |
| case 'U': |
| case 'V': |
| case 'W': |
| case 'X': |
| case 'Y': |
| case 'Z': |
| case '_': |
| case_ident: |
| { |
| // debug writefln( " identifier" ); |
| ubyte c; |
| do { |
| c = *++p; |
| } while (isidchar(c) || (c & 0x80 && isUniAlpha(decodeUTF()))); |
| |
| // sv = stringtable.update((char *)t.ptr, p - t.ptr); |
| char[] tmp; |
| tmp.length = p - t.ptr; |
| memcpy(tmp.ptr, t.ptr, p - t.ptr); |
| Identifier id; |
| Identifier * pid = tmp in stringtable; |
| if (pid) { |
| id = *pid; |
| } |
| |
| if (id is null) { |
| id = new Identifier(tmp, TOK.TOKidentifier); |
| stringtable[tmp] = id; |
| } |
| |
| t.ident = id; |
| t.value = cast(TOK)id.value; |
| anyToken = 1; |
| |
| // if special identifier token |
| if (*t.ptr == '_') { |
| static char date[11 + 1]; |
| static char time[8 + 1]; |
| static char timestamp[24 + 1]; |
| |
| if (!date[0]) { // lazy evaluation |
| //!! |
| /+ |
| * time_t t; |
| * char *p; |
| * .time(&t); |
| * p = ctime(&t); |
| * assert(p); |
| * sprintf(date.ptr, "%.6s %.4s", p + 4, p + 20); |
| * sprintf(time.ptr, "%.8s", p + 11); |
| * sprintf(timestamp.ptr, "%.24s", p); |
| +/ |
| } |
| |
| if (mod && id is Id.FILE) { |
| t.value = TOK.TOKstring; |
| if (loc.filename.length) { |
| t.ustring = loc.filename; |
| } |
| else { |
| t.ustring = mod.ident.toChars(); |
| } |
| goto Llen; |
| } |
| else if (mod && id == Id.LINE) { |
| t.value = TOK.TOKint64v; |
| t.uns64value = loc.linnum; |
| } |
| else if (id == Id.DATE) { |
| t.value = TOK.TOKstring; |
| //! t.ustring = date; |
| goto Llen; |
| } |
| else if (id == Id.TIME) { |
| t.value = TOK.TOKstring; |
| //! t.ustring = time; |
| goto Llen; |
| } |
| else if (id == Id.TIMESTAMP) { |
| t.value = TOK.TOKstring; |
| //! t.ustring = timestamp; |
| Llen: |
| t.postfix = 0; |
| // t.len = strlen((char *)t.ustring); |
| } |
| } |
| //printf("t.value = %d\n",t.value); |
| return; |
| } |
| |
| // comments |
| case '/': |
| p++; |
| switch (*p) { |
| case '=': |
| p++; |
| t.value = TOK.TOKdivass; |
| return; |
| |
| case '*': // '/*' |
| p++; |
| linnum = loc.linnum; |
| while (true) { |
| while (true) { |
| ubyte c = *p; |
| switch (c) { |
| case '/': |
| break; |
| |
| case '\n': |
| loc.linnum++; |
| p++; |
| continue; |
| |
| case '\r': |
| p++; |
| if (*p != '\n') { |
| loc.linnum++; |
| } |
| continue; |
| |
| case 0: |
| case 0x1A: |
| error("unterminated /* */ comment"); |
| p = end; |
| t.value = TOK.TOKeof; |
| return; |
| |
| default: |
| if (c & 0x80) { |
| uint u = decodeUTF(); |
| if (u == PS || u == LS) { |
| loc.linnum++; |
| } |
| } |
| p++; |
| continue; |
| } |
| break; |
| } |
| p++; |
| if (p[-2] == '*' && p - 3 != t.ptr) { |
| break; |
| } |
| } |
| |
| if (commentToken) { |
| t.value = TOK.TOKcomment; |
| return; |
| } |
| // if /** but not /**/ |
| else if (doDocComment && t.ptr[2] == '*' && p - 4 != t.ptr) { |
| getDocComment(t, lastLine == linnum); //! ? |
| } |
| continue; |
| |
| case '/': // do // style comments |
| linnum = loc.linnum; |
| while (1) { |
| ubyte c = *++p; |
| switch (c) { |
| case '\n': |
| break; |
| |
| case '\r': |
| if (p[1] == '\n') { |
| p++; |
| } |
| break; |
| |
| case 0: |
| case 0x1a: |
| if (commentToken) { |
| p = end; |
| t.value = TOK.TOKcomment; |
| return; |
| } |
| if (doDocComment && t.ptr[2] == '/') { |
| getDocComment(t, lastLine == linnum); |
| } |
| p = end; |
| t.value = TOK.TOKeof; |
| return; |
| |
| default: |
| if (c & 0x80) { |
| uint u = decodeUTF(); |
| if (u == PS || u == LS) { |
| break; |
| } |
| } |
| continue; |
| } |
| break; |
| } |
| |
| if (commentToken) { |
| p++; |
| loc.linnum++; |
| t.value = TOK.TOKcomment; |
| return; |
| } |
| if (doDocComment && t.ptr[2] == '/') { |
| getDocComment(t, lastLine == linnum); |
| } |
| |
| p++; |
| loc.linnum++; |
| continue; |
| |
| case '+': |
| { |
| int nest; |
| linnum = loc.linnum; |
| p++; |
| nest = 1; |
| while (1) { |
| ubyte c = *p; |
| switch (c) { |
| case '/': |
| p++; |
| if (*p == '+') { |
| p++; |
| nest++; |
| } |
| continue; |
| |
| case '+': |
| p++; |
| if (*p == '/') { |
| p++; |
| if (--nest == 0) { |
| break; |
| } |
| } |
| continue; |
| |
| case '\r': |
| p++; |
| if (*p != '\n') { |
| loc.linnum++; |
| } |
| continue; |
| |
| case '\n': |
| loc.linnum++; |
| p++; |
| continue; |
| |
| case 0: |
| case 0x1A: |
| error("unterminated /+ +/ comment"); |
| p = end; |
| t.value = TOK.TOKeof; |
| return; |
| |
| default: |
| if (c & 0x80) { |
| uint u = decodeUTF(); |
| if (u == PS || u == LS) { |
| loc.linnum++; |
| } |
| } |
| p++; |
| continue; |
| } |
| break; |
| } |
| if (commentToken) { |
| t.value = TOK.TOKcomment; |
| return; |
| } |
| if (doDocComment && t.ptr[2] == '+' && p - 4 != t.ptr) { |
| // if /++ but not /++/ |
| getDocComment(t, lastLine == linnum); |
| } |
| continue; |
| } |
| |
| default: |
| break; |
| } |
| t.value = TOK.TOKdiv; |
| return; |
| |
| case '.': |
| p++; |
| if (isdigit(*p)) { |
| p--; |
| t.value = inreal(t); |
| } |
| else if (p[0] == '.') { |
| if (p[1] == '.') { |
| p += 2; |
| t.value = TOK.TOKdotdotdot; |
| } |
| else { |
| p++; |
| t.value = TOK.TOKslice; |
| } |
| } |
| else { |
| t.value = TOK.TOKdot; |
| } |
| return; |
| |
| case '&': |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKandass; |
| } |
| else if (*p == '&') { |
| p++; |
| t.value = TOK.TOKandand; |
| } |
| else { |
| t.value = TOK.TOKand; |
| } |
| return; |
| |
| // |, ||, |= |
| case '|': |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKorass; |
| } |
| else if (*p == '|') { |
| p++; |
| t.value = TOK.TOKoror; |
| } |
| else { |
| t.value = TOK.TOKor; |
| } |
| return; |
| |
| case '-': |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKminass; |
| } |
| else if (*p == '-') { |
| p++; |
| t.value = TOK.TOKminusminus; |
| } |
| else { |
| t.value = TOK.TOKmin; |
| } |
| return; |
| |
| // +, +=, ++ |
| case '+': |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKaddass; // += |
| } |
| else if (*p == '+') { |
| p++; |
| t.value = TOK.TOKplusplus; // ++ |
| } |
| else { |
| t.value = TOK.TOKadd; // + |
| } |
| return; |
| |
| // <, <=, <<=, <<, <>=, <> |
| case '<': |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKle; // <= |
| } |
| else if (*p == '<') { |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKshlass; // <<= |
| } |
| else { |
| t.value = TOK.TOKshl; // << |
| } |
| } |
| else if (*p == '>') { |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKleg; // <>= |
| } |
| else { |
| t.value = TOK.TOKlg; // <> |
| } |
| } |
| else { |
| t.value = TOK.TOKlt; // < |
| } |
| return; |
| |
| // >, >>, >>>, >=, >>=, >>>= |
| case '>': |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKge; // >= |
| } |
| else if (*p == '>') { |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKshrass; // >>= |
| } |
| else if (*p == '>') { |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKushrass; // >>>= |
| } |
| else { |
| t.value = TOK.TOKushr; // >>> |
| } |
| } |
| else { |
| t.value = TOK.TOKshr; // >> |
| } |
| } |
| else { |
| t.value = TOK.TOKgt; // > |
| } |
| return; |
| |
| case '!': |
| p++; |
| if (*p == '=') { |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKnotidentity; // !== |
| } |
| else { |
| t.value = TOK.TOKnotequal; // != |
| } |
| } |
| else if (*p == '<') { |
| p++; |
| if (*p == '>') { |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKunord; // !<>= |
| } |
| else { |
| t.value = TOK.TOKue; // !<> |
| } |
| } |
| else if (*p == '=') { |
| p++; |
| t.value = TOK.TOKug; // !<= |
| } |
| else { |
| t.value = TOK.TOKuge; // !< |
| } |
| } |
| else if (*p == '>') { |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKul; // !>= |
| } |
| else { |
| t.value = TOK.TOKule; // !> |
| } |
| } |
| else { |
| t.value = TOK.TOKnot; // ! |
| } |
| return; |
| |
| case '=': |
| p++; |
| if (*p == '=') { |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKidentity; // === |
| } |
| else { |
| t.value = TOK.TOKequal; // == |
| } |
| } |
| else { |
| t.value = TOK.TOKassign; // = |
| } |
| return; |
| |
| case '~': |
| p++; |
| if (*p == '=') { |
| p++; |
| t.value = TOK.TOKcatass; // ~= |
| } |
| else { |
| t.value = TOK.TOKtilde; // ~ |
| } |
| return; |
| |
| // SINGLE |
| case '(': p++; t.value = TOK.TOKlparen; return; |
| |
| case ')': p++; t.value = TOK.TOKrparen; return; |
| |
| case '[': p++; t.value = TOK.TOKlbracket; return; |
| |
| case ']': p++; t.value = TOK.TOKrbracket; return; |
| |
| case '{': p++; t.value = TOK.TOKlcurly; return; |
| |
| case '}': p++; t.value = TOK.TOKrcurly; return; |
| |
| case '?': p++; t.value = TOK.TOKquestion; return; |
| |
| case ',': p++; t.value = TOK.TOKcomma; return; |
| |
| case ';': p++; t.value = TOK.TOKsemicolon; return; |
| |
| case ':': p++; t.value = TOK.TOKcolon; return; |
| |
| case '$': p++; t.value = TOK.TOKdollar; return; |
| |
| // DOUBLE |
| case '*': p++; if (*p == '=') { |
| p++; t.value = TOK.TOKmulass; |
| } |
| else { |
| t.value = TOK.TOKmul; |
| } return; |
| |
| case '%': p++; if (*p == '=') { |
| p++; t.value = TOK.TOKmodass; |
| } |
| else { |
| t.value = TOK.TOKmod; |
| } return; |
| |
| case '^': p++; if (*p == '=') { |
| p++; t.value = TOK.TOKxorass; |
| } |
| else { |
| t.value = TOK.TOKxor; |
| } return; |
| |
| // removed 148 case '~': p++; if( *p == '=' ) { p++; t.value = TOK.TOKcatass; } else t.value = TOK.TOKtilde; return; |
| |
| |
| case '#': |
| p++; |
| Pragma(); |
| continue; |
| |
| default: |
| { |
| debug writefln(" default char"); |
| ubyte c = *p; |
| if (c & 0x80) { |
| uint u = decodeUTF(); |
| // Check for start of unicode identifier |
| if (isUniAlpha(u)) { |
| goto case_ident; |
| } |
| |
| if (u == PS || u == LS) { |
| loc.linnum++; |
| p++; |
| continue; |
| } |
| } |
| if (isprint(c)) { |
| error("unsupported char '%s'", cast(char)c); |
| } |
| else { |
| error("unsupported char 0x%02x", cast(ubyte)c); |
| } |
| p++; |
| continue; |
| } |
| } |
| } |
| } |
| |
| |
| |
| // Parse escape sequence. |
| uint escapeSequence() { |
| uint c; |
| int n; |
| int ndigits; |
| |
| c = *p; |
| switch (c) { |
| case '\'': |
| case '"': |
| case '?': |
| case '\\': |
| Lconsume: |
| p++; |
| break; |
| |
| case 'a': c = 7; goto Lconsume; |
| |
| case 'b': c = 8; goto Lconsume; |
| |
| case 'f': c = 12; goto Lconsume; |
| |
| case 'n': c = 10; goto Lconsume; |
| |
| case 'r': c = 13; goto Lconsume; |
| |
| case 't': c = 9; goto Lconsume; |
| |
| case 'v': c = 11; goto Lconsume; |
| |
| case 'u': |
| ndigits = 4; |
| goto Lhex; |
| |
| case 'U': |
| ndigits = 8; |
| goto Lhex; |
| |
| case 'x': |
| ndigits = 2; |
| Lhex: |
| p++; |
| c = *p; |
| if (ishex(c)) { |
| uint v; |
| n = 0; |
| v = 0; |
| while (1) { |
| if (isdigit(c)) { |
| c -= '0'; |
| } |
| else if (islower(c)) { |
| c -= 'a' - 10; |
| } |
| else { |
| c -= 'A' - 10; |
| } |
| v = v * 16 + c; |
| c = *++p; |
| if (++n == ndigits) { |
| break; |
| } |
| if (!ishex(c)) { |
| error("escape hex sequence has %d hex digits instead of %d", n, ndigits); |
| break; |
| } |
| } |
| //! if( ndigits != 2 && !utf_isValidDchar(v)) |
| //! error("invalid UTF character \\U%08x", v); |
| c = v; |
| } |
| else { |
| error("undefined escape hex sequence \\%s\n", c); |
| } |
| break; |
| |
| case '&': // named character entity |
| for (ubyte *idstart = ++p; 1; p++) { |
| switch (*p) { |
| case ';': |
| //!!! |
| /+ |
| * c = HtmlNamedEntity(idstart, p - idstart); |
| * if( c == ~0 ) |
| * { |
| * error("unnamed character entity &%.*s;", p - idstart, idstart); |
| * c = ' '; |
| * } |
| * |
| * p++; |
| +/ |
| break; |
| |
| default: |
| if (isalpha(*p) || (p != idstart + 1 && isdigit(*p))) { |
| continue; |
| } |
| error("unterminated named entity"); |
| break; |
| } |
| break; |
| } |
| break; |
| |
| case 0: |
| case 0x1a: // end of file |
| c = '\\'; |
| break; |
| |
| default: |
| if (isoctal(c)) { |
| ubyte v; |
| n = 0; |
| do { |
| v = v * 8 + (c - '0'); |
| c = *++p; |
| } while (++n < 3 && isoctal(c)); |
| c = v; |
| } |
| else { |
| error("undefined escape sequence \\%s\n", c); |
| } |
| break; |
| } |
| return(c); |
| } |
| |
| /************************************** |
| */ |
| |
| TOK wysiwygStringConstant(Token *t, int tc) { |
| uint c; |
| Loc start = loc; |
| |
| p++; |
| stringbuffer.offset = 0; |
| while (1) { |
| c = *p++; |
| switch (c) { |
| case '\n': |
| loc.linnum++; |
| break; |
| |
| case '\r': |
| if (*p == '\n') { |
| continue; // ignore |
| } |
| c = '\n'; // treat EndOfLine as \n character |
| loc.linnum++; |
| break; |
| |
| case 0: |
| case 0x1a: |
| error("unterminated string constant starting at %s", start.toChars()); |
| t.ustring = ""; |
| t.postfix = 0; |
| return(TOK.TOKstring); |
| |
| case '"': |
| case '`': |
| if (c == tc) { |
| // t.len = stringbuffer.offset; |
| stringbuffer.write(cast(byte)0); |
| t.ustring = stringbuffer.toString; |
| // t.ustring = (ubyte *)mem.malloc(stringbuffer.offset); |
| // memcpy(t.ustring, stringbuffer.data, stringbuffer.offset); |
| stringPostfix(t); |
| return(TOK.TOKstring); |
| } |
| break; |
| |
| default: |
| if (c & 0x80) { |
| p--; |
| uint u = decodeUTF(); |
| p++; |
| if (u == PS || u == LS) { |
| loc.linnum++; |
| } |
| stringbuffer.write(u); |
| continue; |
| } |
| break; |
| } |
| stringbuffer.write(c); |
| } |
| } |
| |
| /************************************** |
| * Lex hex strings: |
| * x"0A ae 34FE BD" |
| */ |
| |
| TOK hexStringConstant(Token *t) { |
| uint c; |
| Loc start = loc; |
| uint n = 0; |
| uint v; |
| |
| p++; |
| stringbuffer.offset = 0; |
| while (1) { |
| c = *p++; |
| switch (c) { |
| case ' ': |
| case '\t': |
| case '\v': |
| case '\f': |
| continue; // skip white space |
| |
| case '\r': |
| if (*p == '\n') { |
| continue; // ignore |
| } |
| |
| // Treat isolated '\r' as if it were a '\n' |
| case '\n': |
| loc.linnum++; |
| continue; |
| |
| case 0: |
| case 0x1a: |
| error("unterminated string constant starting at %s", start.toChars()); |
| t.ustring = ""; |
| t.postfix = 0; |
| return(TOK.TOKstring); |
| |
| case '"': |
| if (n & 1) { |
| error("odd number (%d) of hex characters in hex string", n); |
| stringbuffer.write(v); |
| } |
| // t.len = stringbuffer.offset; |
| // stringbuffer.write(cast(byte)0); |
| t.ustring = stringbuffer.toString; |
| // t.ustring = (ubyte *)mem.malloc(stringbuffer.offset); |
| // memcpy(t.ustring, stringbuffer.data, stringbuffer.offset); |
| stringPostfix(t); |
| return(TOK.TOKstring); |
| |
| default: |
| if (c >= '0' && c <= '9') { |
| c -= '0'; |
| } |
| else if (c >= 'a' && c <= 'f') { |
| c -= 'a' - 10; |
| } |
| else if (c >= 'A' && c <= 'F') { |
| c -= 'A' - 10; |
| } |
| else if (c & 0x80) { |
| p--; |
| uint u = decodeUTF(); |
| p++; |
| if (u == PS || u == LS) { |
| loc.linnum++; |
| } |
| else { |
| error("non-hex character \\u%x", u); |
| } |
| } |
| else { |
| error("non-hex character '%s'", c); |
| } |
| if (n & 1) { |
| v = (v << 4) | c; |
| stringbuffer.write(v); |
| } |
| else { |
| v = c; |
| } |
| n++; |
| break; |
| } |
| } |
| } |
| |
| /************************************** |
| */ |
| |
| TOK escapeStringConstant(Token *t, int wide) { |
| uint c; |
| Loc start = loc; |
| |
| p++; |
| stringbuffer.offset = 0; |
| // debug writefln( "escape string constant: %s", std.string.toString( cast(char*)p ) ); |
| while (1) { |
| c = *p++; |
| switch (c) { |
| case '\\': |
| switch (*p) { |
| case 'u': |
| case 'U': |
| case '&': |
| c = escapeSequence(); |
| stringbuffer.write(c); |
| continue; |
| |
| default: |
| c = escapeSequence(); |
| break; |
| } |
| break; |
| |
| case '\n': |
| loc.linnum++; |
| break; |
| |
| case '\r': |
| if (*p == '\n') { |
| continue; // ignore |
| } |
| c = '\n'; // treat EndOfLine as \n character |
| loc.linnum++; |
| break; |
| |
| case '"': |
| // writefln( "end of string: ", stringbuffer.toString ); |
| t.ustring = stringbuffer.toString().dup; |
| // t.len = stringbuffer.offset; |
| // stringbuffer.write(cast(byte)0); |
| // t.ustring = (ubyte *)mem.malloc(stringbuffer.offset); |
| // memcpy(t.ustring, stringbuffer.data, stringbuffer.offset); |
| stringPostfix(t); |
| |
| return(TOK.TOKstring); |
| |
| case 0: |
| case 0x1a: |
| p--; |
| error("unterminated string constant starting at %s", start.toChars()); |
| t.ustring = ""; |
| // t.len = 0; |
| t.postfix = 0; |
| return(TOK.TOKstring); |
| |
| default: |
| if (c & 0x80) { |
| p--; |
| c = decodeUTF(); |
| if (c == LS || c == PS) { |
| c = '\n'; |
| loc.linnum++; |
| } |
| p++; |
| stringbuffer.write(cast(char)c); |
| continue; |
| } |
| break; |
| } |
| stringbuffer.write(cast(char)c); |
| // writefln( stringbuffer.toString ); |
| } |
| } |
| |
| //************************************** |
| TOK charConstant(Token *t, int wide) { |
| uint c; |
| TOK tk = TOK.TOKcharv; |
| |
| //printf("Lexer.charConstant\n"); |
| p++; |
| c = *p++; |
| switch (c) { |
| case '\\': |
| switch (*p) { |
| case 'u': |
| t.uns64value = escapeSequence(); |
| tk = TOK.TOKwcharv; |
| break; |
| |
| case 'U': |
| case '&': |
| t.uns64value = escapeSequence(); |
| tk = TOK.TOKdcharv; |
| break; |
| |
| default: |
| t.uns64value = escapeSequence(); |
| break; |
| } |
| break; |
| |
| case '\n': |
| L1: |
| loc.linnum++; |
| |
| case '\r': |
| case 0: |
| case 0x1a: |
| case '\'': |
| error("unterminated character constant"); |
| return(tk); |
| |
| default: |
| if (c & 0x80) { |
| p--; |
| c = decodeUTF(); |
| p++; |
| if (c == LS || c == PS) { |
| goto L1; |
| } |
| if (c < 0xd800 || (c >= 0xe000 && c < 0xfffe)) { |
| tk = TOK.TOKwcharv; |
| } |
| else { |
| tk = TOK.TOKdcharv; |
| } |
| } |
| t.uns64value = c; |
| break; |
| } |
| |
| if (*p != '\'') { |
| error("unterminated character constant"); |
| return(tk); |
| } |
| p++; |
| return(tk); |
| } |
| |
| // Get postfix of string literal. |
| void stringPostfix(Token *t) { |
| switch (*p) { |
| case 'c': |
| case 'w': |
| case 'd': |
| t.postfix = *p; |
| p++; |
| break; |
| |
| default: |
| t.postfix = 0; |
| break; |
| } |
| } |
| |
| /*************************************** |
| * Read \u or \U unicode sequence |
| * Input: |
| * u 'u' or 'U' |
| */ |
| /* |
| * uint Wchar(uint u) |
| * { |
| * uint value; |
| * uint n; |
| * ubyte c; |
| * uint nchars; |
| * |
| * nchars = (u == 'U') ? 8 : 4; |
| * value = 0; |
| * for (n = 0; 1; n++) |
| * { |
| * ++p; |
| * if( n == nchars) |
| * break; |
| * c = *p; |
| * if( !ishex(c)) |
| * { |
| * error("\\%s sequence must be followed by %d hex characters", u, nchars); |
| * break; |
| * } |
| * if( isdigit(c)) |
| * c -= '0'; |
| * else if( islower(c)) |
| * c -= 'a' - 10; |
| * else |
| * c -= 'A' - 10; |
| * value <<= 4; |
| * value |= c; |
| * } |
| * return value; |
| * } |
| */ |
| |
| /************************************** |
| * Read in a number. |
| * If it's an integer, store it in tok.TKutok.Vlong. |
| * integers can be decimal, octal or hex |
| * Handle the suffixes U, UL, LU, L, etc. |
| * If it's double, store it in tok.TKutok.Vdouble. |
| * Returns: |
| * TKnum |
| * TKdouble,... |
| */ |
| |
| TOK number(Token *t) { |
| //debug writefln("Lexer.number()"); |
| // We use a state machine to collect numbers |
| enum STATE { |
| STATE_initial, |
| STATE_0, |
| STATE_decimal, |
| STATE_octal, |
| STATE_octale, |
| STATE_hex, |
| STATE_binary, |
| STATE_hex0, |
| STATE_binary0, |
| STATE_hexh, |
| STATE_error |
| } |
| |
| enum FLAGS { |
| FLAGS_decimal = 1, // decimal |
| FLAGS_unsigned = 2, // u or U suffix |
| FLAGS_long = 4, // l or L suffix |
| } |
| FLAGS flags = FLAGS.FLAGS_decimal; |
| |
| int i; |
| TOK result; |
| int base; |
| |
| stringbuffer.offset = 0; |
| // stringbuffer.data = null; |
| STATE state = STATE.STATE_initial; |
| ubyte * start = p; |
| |
| TOK _isreal() { |
| p = start; |
| return(inreal(t)); |
| } |
| |
| while (true) { |
| char c = cast(char)*p; |
| switch (state) { |
| case STATE.STATE_initial: // opening state |
| if (c == '0') { |
| state = STATE.STATE_0; |
| } |
| else { |
| state = STATE.STATE_decimal; |
| } |
| break; |
| |
| case STATE.STATE_0: |
| flags = cast(FLAGS)(flags & ~FLAGS.FLAGS_decimal); |
| switch (c) { |
| // #if ZEROH |
| // case 'H': // 0h |
| // case 'h': |
| // goto hexh; |
| // #endif |
| case 'X': |
| case 'x': |
| state = STATE.STATE_hex0; |
| break; |
| |
| case '.': |
| if (p[1] == '.') { // .. is a separate token |
| goto done; |
| } |
| |
| case 'i': |
| case 'f': |
| case 'F': |
| goto _Real; |
| |
| // #if ZEROH |
| // case 'E': |
| // case 'e': |
| // goto case_hex; |
| // #endif |
| case 'B': |
| case 'b': |
| state = STATE.STATE_binary0; |
| break; |
| |
| case '0': |
| case '1': |
| case '2': |
| case '3': |
| case '4': |
| case '5': |
| case '6': |
| case '7': |
| state = STATE.STATE_octal; |
| break; |
| |
| // #if ZEROH |
| // case '8': case '9': case 'A': |
| // case 'C': case 'D': case 'F': |
| // case 'a': case 'c': case 'd': case 'f': |
| // case_hex: |
| // state = STATE.STATE_hexh; |
| // break; |
| // #endif |
| case '_': |
| state = STATE.STATE_octal; |
| p++; |
| continue; |
| |
| default: |
| goto done; |
| } |
| break; |
| |
| case STATE.STATE_decimal: // reading decimal number |
| |
| // if its not a digit - decimal complete or not a decimal |
| if (!isdigit(c)) { |
| // debug writefln( "\tnon-digit( %s )", c ); |
| // #if ZEROH |
| // if( ishex(c) || c == 'H' || c == 'h' ) |
| // goto hexh; |
| // #endif |
| //! wtf ? |
| // ignore embedded _ |
| if (c == '_') { |
| p++; |
| continue; |
| } |
| |
| // check decimal point - make real |
| if (c == '.' && p[1] != '.') { |
| goto _Real; |
| } |
| |
| // check for mantra - make real |
| if (c == 'i' || c == 'f' || c == 'F' || c == 'e' || c == 'E') { |
| _Real: // It's a real number. Back up and rescan as a real |
| p = start; |
| return(inreal(t)); |
| } |
| |
| goto done; |
| } |
| break; |
| |
| case STATE.STATE_hex0: // reading hex number |
| case STATE.STATE_hex: |
| if (!ishex(c)) { |
| if (c == '_') { // ignore embedded _ |
| p++; |
| continue; |
| } |
| if (c == '.' && p[1] != '.') { |
| goto _Real; |
| } |
| if (c == 'P' || c == 'p' || c == 'i') { |
| goto _Real; |
| } |
| if (state == STATE.STATE_hex0) { |
| error("Hex digit expected, not '%s'", c); |
| } |
| goto done; |
| } |
| state = STATE.STATE_hex; |
| break; |
| |
| // #if ZEROH |
| // hexh: |
| // state = STATE.STATE_hexh; |
| // |
| // case STATE.STATE_hexh: // parse numbers like 0FFh |
| // if( !ishex(c)) |
| // { |
| // if( c == 'H' || c == 'h') |
| // { |
| // p++; |
| // base = 16; |
| // goto done; |
| // } |
| // else |
| // { |
| // // Check for something like 1E3 or 0E24 |
| // if( memchr(stringbuffer.data.ptr, 'E', stringbuffer.offset) || memchr( stringbuffer.data.ptr, 'e', stringbuffer.offset)) |
| // goto _Real; |
| // error("Hex digit expected, not '%s'", c); |
| // goto done; |
| // } |
| // } |
| // break; |
| // #endif |
| |
| case STATE.STATE_octal: // reading octal number |
| case STATE.STATE_octale: // reading octal number with non-octal digits |
| if (!isoctal(c)) { |
| // #if ZEROH |
| // if( ishex(c) || c == 'H' || c == 'h' ) |
| // goto hexh; |
| // #endif |
| if (c == '_') { // ignore embedded _ |
| p++; |
| continue; |
| } |
| if (c == '.' && p[1] != '.') { |
| goto _Real; |
| } |
| if (c == 'i') { |
| goto _Real; |
| } |
| if (isdigit(c)) { |
| state = STATE.STATE_octale; |
| } |
| else { |
| goto done; |
| } |
| } |
| break; |
| |
| case STATE.STATE_binary0: // starting binary number |
| case STATE.STATE_binary: // reading binary number |
| if (c != '0' && c != '1') { |
| // #if ZEROH |
| // if( ishex(c) || c == 'H' || c == 'h' ) |
| // goto hexh; |
| // #endif |
| if (c == '_') { // ignore embedded _ |
| p++; |
| continue; |
| } |
| if (state == STATE.STATE_binary0) { |
| error("binary digit expected"); |
| state = STATE.STATE_error; |
| break; |
| } |
| else { |
| goto done; |
| } |
| } |
| state = STATE.STATE_binary; |
| break; |
| |
| case STATE.STATE_error: // for error recovery |
| if (!isdigit(c)) { // scan until non-digit |
| goto done; |
| } |
| break; |
| |
| default: |
| assert(0); |
| } |
| stringbuffer.write(cast(char)c); |
| p++; |
| } |
| done: |
| stringbuffer.write(cast(char)0); // terminate string |
| |
| // debug writefln( "\tdigit complete( %s )", stringbuffer.toString ); |
| |
| if (state == STATE.STATE_octale) { |
| error("Octal digit expected"); |
| } |
| |
| uinteger_t n; // unsigned >=64 bit integer type |
| |
| if (stringbuffer.offset == 2 && (state == STATE.STATE_decimal || state == STATE.STATE_0)) { |
| n = stringbuffer.data[0] - '0'; |
| } |
| else { |
| // Convert string to integer |
| char* p = cast(char*)stringbuffer.data.ptr; |
| int r = 10; |
| int d; |
| if (*p == '0') { |
| if (p[1] == 'x' || p[1] == 'X') { |
| // "0x#" |
| p += 2; |
| r = 16; |
| } |
| else if (p[1] == 'b' || p[1] == 'B') { |
| // "0b#" - binary |
| p += 2; |
| r = 2; |
| } |
| else if (isdigit(p[1])) { |
| p += 1; |
| r = 8; |
| } |
| } |
| |
| n = 0; |
| |
| while (true) { |
| if (*p >= '0' && *p <= '9') { |
| d = *p - '0'; |
| } |
| else if (*p >= 'a' && *p <= 'z') { |
| d = *p - 'a' + 10; |
| } |
| else if (*p >= 'A' && *p <= 'Z') { |
| d = *p - 'A' + 10; |
| } |
| else { |
| break; |
| } |
| |
| if (d >= r) { |
| break; |
| } |
| |
| if (n * r + d < n) { |
| error("integer overflow"); |
| break; |
| } |
| |
| n = n * r + d; |
| p++; |
| } |
| |
| // if n needs more than 64 bits |
| if (n.sizeof > 8 && n > 0xffffffffffffffffL) { |
| error("integer overflow"); |
| } |
| } |
| |
| // Parse trailing 'u', 'U', 'l' or 'L' in any combination |
| while (true) { |
| ubyte f; |
| switch (*p) { |
| case 'U': |
| case 'u': |
| f = FLAGS.FLAGS_unsigned; |
| goto L1; |
| |
| case 'L': |
| case 'l': |
| f = FLAGS.FLAGS_long; |
| L1: |
| p++; |
| if (flags & f) { |
| error("unrecognized token"); |
| } |
| flags = cast(FLAGS)(flags | f); |
| continue; |
| |
| default: |
| break; |
| } |
| break; |
| } |
| |
| switch (flags) { |
| case 0: |
| /* Octal or Hexadecimal constant. |
| * First that fits: int, uint, long, ulong |
| */ |
| if (n & 0x8000000000000000L) { |
| result = TOK.TOKuns64v; |
| } |
| else if (n & 0xffffffff00000000L) { |
| result = TOK.TOKint64v; |
| } |
| else if (n & 0x80000000) { |
| result = TOK.TOKuns32v; |
| } |
| else { |
| result = TOK.TOKint32v; |
| } |
| break; |
| |
| case FLAGS.FLAGS_decimal: |
| /* First that fits: int, long, long long |
| */ |
| if (n & 0x8000000000000000L) { |
| error("signed integer overflow"); |
| result = TOK.TOKuns64v; |
| } |
| else if (n & 0xffffffff80000000L) { |
| result = TOK.TOKint64v; |
| } |
| else { |
| result = TOK.TOKint32v; |
| } |
| break; |
| |
| case FLAGS.FLAGS_unsigned: |
| case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned: |
| /* First that fits: uint, ulong |
| */ |
| if (n & 0xffffffff00000000L) { |
| result = TOK.TOKuns64v; |
| } |
| else { |
| result = TOK.TOKuns32v; |
| } |
| break; |
| |
| case FLAGS.FLAGS_decimal | FLAGS.FLAGS_long: |
| if (n & 0x8000000000000000L) { |
| error("signed integer overflow"); |
| result = TOK.TOKuns64v; |
| } |
| else { |
| result = TOK.TOKint64v; |
| } |
| break; |
| |
| case FLAGS.FLAGS_long: |
| if (n & 0x8000000000000000L) { |
| result = TOK.TOKuns64v; |
| } |
| else { |
| result = TOK.TOKint64v; |
| } |
| break; |
| |
| case FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long: |
| case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long: |
| result = TOK.TOKuns64v; |
| break; |
| |
| default: |
| debug writefln("%x", flags); |
| assert(0); |
| } |
| t.uns64value = n; |
| return(result); |
| } |
| |
| /************************************** |
| * Read in characters, converting them to real. |
| * Bugs: |
| * Exponent overflow not detected. |
| * Too much requested precision is not detected. |
| */ |
| |
| TOK inreal(Token *t) { |
| int dblstate; |
| uint c; |
| char hex; // is this a hexadecimal-floating-constant? |
| TOK result; |
| |
| //printf("Lexer.inreal()\n"); |
| stringbuffer.offset = 0; |
| dblstate = 0; |
| hex = 0; |
| Lnext: |
| while (1) { |
| // Get next char from input |
| c = *p++; |
| //printf("dblstate = %d, c = '%s'\n", dblstate, c); |
| while (1) { |
| switch (dblstate) { |
| case 0: // opening state |
| if (c == '0') { |
| dblstate = 9; |
| } |
| else if (c == '.') { |
| dblstate = 3; |
| } |
| else { |
| dblstate = 1; |
| } |
| break; |
| |
| case 9: |
| dblstate = 1; |
| if (c == 'X' || c == 'x') { |
| hex++; |
| break; |
| } |
| |
| case 1: // digits to left of . |
| case 3: // digits to right of . |
| case 7: // continuing exponent digits |
| if (!isdigit(c) && !(hex && isxdigit(c))) { |
| if (c == '_') { |
| goto Lnext; // ignore embedded '_' |
| } |
| dblstate++; |
| continue; |
| } |
| break; |
| |
| case 2: // no more digits to left of . |
| if (c == '.') { |
| dblstate++; |
| break; |
| } |
| |
| case 4: // no more digits to right of . |
| if ((c == 'E' || c == 'e') || hex && (c == 'P' || c == 'p')) { |
| dblstate = 5; |
| hex = 0; // exponent is always decimal |
| break; |
| } |
| if (hex) { |
| error("binary-exponent-part required"); |
| } |
| goto done; |
| |
| case 5: // looking immediately to right of E |
| dblstate++; |
| if (c == '-' || c == '+') { |
| break; |
| } |
| |
| case 6: // 1st exponent digit expected |
| if (!isdigit(c)) { |
| error("exponent expected"); |
| } |
| dblstate++; |
| break; |
| |
| case 8: // past end of exponent digits |
| goto done; |
| } |
| break; |
| } |
| stringbuffer.write(c); |
| } |
| done: |
| p--; |
| |
| stringbuffer.write(cast(byte)0); |
| |
| // #if _WIN32 && __DMC__ |
| char *save = __locale_decpoint; |
| __locale_decpoint = "."; |
| // #endif |
| t.float80value = strtold(cast(char *)stringbuffer.data.ptr, null); |
| errno = 0; |
| switch (*p) { |
| case 'F': |
| case 'f': |
| strtof(cast(char *)stringbuffer.data.ptr, null); |
| result = TOK.TOKfloat32v; |
| p++; |
| break; |
| |
| default: |
| strtod(cast(char *)stringbuffer.data.ptr, null); |
| result = TOK.TOKfloat64v; |
| break; |
| |
| case 'L': |
| case 'l': |
| result = TOK.TOKfloat80v; |
| p++; |
| break; |
| } |
| if (*p == 'i' || *p == 'I') { |
| p++; |
| switch (result) { |
| case TOK.TOKfloat32v: |
| result = TOK.TOKimaginary32v; |
| break; |
| |
| case TOK.TOKfloat64v: |
| result = TOK.TOKimaginary64v; |
| break; |
| |
| case TOK.TOKfloat80v: |
| result = TOK.TOKimaginary80v; |
| break; |
| } |
| } |
| // #if _WIN32 && __DMC__ |
| __locale_decpoint = save; |
| // #endif |
| if (errno == ERANGE) { |
| error("number is not representable"); |
| } |
| return(result); |
| } |
| |
| |
| |
| |
| /********************************************* |
| * Do pragma. |
| * Currently, the only pragma supported is: |
| * #line linnum [filespec] |
| */ |
| |
| void Pragma() { |
| Token tok; |
| int linnum; |
| |
| char[] filespec; |
| Loc loc = this.loc; |
| |
| scan(&tok); |
| |
| if (tok.value != TOK.TOKidentifier || tok.ident != Id.line) { |
| goto Lerr; |
| } |
| |
| scan(&tok); |
| if (tok.value == TOK.TOKint32v || tok.value == TOK.TOKint64v) { |
| linnum = tok.uns64value - 1; |
| } |
| else { |
| goto Lerr; |
| } |
| |
| while (1) { |
| switch (*p) { |
| case 0: |
| case 0x1a: |
| case '\n': |
| Lnewline: |
| this.loc.linnum = linnum; |
| if (filespec.length) { |
| this.loc.filename = filespec; |
| } |
| return; |
| |
| case '\r': |
| p++; |
| if (*p != '\n') { |
| p--; |
| goto Lnewline; |
| } |
| continue; |
| |
| case ' ': |
| case '\t': |
| case '\v': |
| case '\f': |
| p++; |
| continue; // skip white space |
| |
| case '_': |
| if (mod && memcmp(p, cast(char*)"__FILE__", 8) == 0) { |
| p += 8; |
| //! filespec = mem.strdup(loc.filename ? loc.filename : mod.ident.toChars()); |
| } |
| continue; |
| |
| case '"': |
| if (filespec) { |
| goto Lerr; |
| } |
| stringbuffer.offset = 0; |
| p++; |
| while (1) { |
| uint c; |
| c = *p; |
| switch (c) { |
| case '\n': |
| case '\r': |
| case 0: |
| case 0x1a: |
| goto Lerr; |
| |
| case '"': |
| stringbuffer.write(cast(byte)0); |
| // filespec = mem.strdup((char *)stringbuffer.data); |
| filespec = stringbuffer.toString.dup; |
| p++; |
| break; |
| |
| default: |
| if (c & 0x80) { |
| uint u = decodeUTF(); |
| if (u == PS || u == LS) { |
| goto Lerr; |
| } |
| } |
| stringbuffer.write(c); |
| p++; |
| continue; |
| } |
| break; |
| } |
| continue; |
| |
| default: |
| if (*p & 0x80) { |
| uint u = decodeUTF(); |
| if (u == PS || u == LS) { |
| goto Lnewline; |
| } |
| } |
| goto Lerr; |
| } |
| } |
| |
| Lerr: |
| errorLoc(loc, "#line integer [\"filespec\"]\\n expected"); |
| } |
| |
| |
| |
| /*************************************************** |
| * Parse doc comment embedded between t.ptr and p. |
| * Remove trailing blanks and tabs from lines. |
| * Replace all newlines with \n. |
| * Remove leading comment character from each line. |
| * Decide if it's a lineComment or a blockComment. |
| * Append to previous one for this token. |
| */ |
| |
| void getDocComment(Token *t, uint lineComment) { |
| auto OutBuffer buf = new OutBuffer; |
| ubyte ct = t.ptr[2]; |
| ubyte *q = t.ptr + 3; // start of comment text |
| int linestart = 0; |
| |
| ubyte *qend = p; |
| |
| if (ct == '*' || ct == '+') { |
| qend -= 2; |
| } |
| |
| // Scan over initial row of ****'s or ++++'s or ////'s |
| for (; q < qend; q++) { |
| if (*q != ct) { |
| break; |
| } |
| } |
| |
| // Remove trailing row of ****'s or ++++'s |
| if (ct != '/') { |
| for (; q < qend; qend--) { |
| if (qend[-1] != ct) { |
| break; |
| } |
| } |
| } |
| |
| for (; q < qend; q++) { |
| ubyte c = *q; |
| |
| switch (c) { |
| case '*': |
| case '+': |
| if (linestart && c == ct) { |
| linestart = 0; |
| // Trim preceding whitespace up to preceding \n |
| while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t')) { |
| buf.offset--; |
| } |
| continue; |
| } |
| break; |
| |
| case ' ': |
| case '\t': |
| break; |
| |
| case '\r': |
| if (q[1] == '\n') { |
| continue; // skip the \r |
| } |
| goto Lnewline; |
| |
| default: |
| if (c == 226) { |
| // If LS or PS |
| if (q[1] == 128 && |
| (q[2] == 168 || q[2] == 169)) { |
| q += 2; |
| goto Lnewline; |
| } |
| } |
| linestart = 0; |
| break; |
| |
| Lnewline: |
| c = '\n'; // replace all newlines with \n |
| |
| case '\n': |
| linestart = 1; |
| |
| // Trim trailing whitespace |
| while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t')) { |
| buf.offset--; |
| } |
| |
| break; |
| } |
| buf.write(c); |
| } |
| |
| // Always end with a newline |
| if (!buf.offset || buf.data[buf.offset - 1] != '\n') { |
| buf.writenl(); |
| } |
| |
| //buf.write(cast(char)0); |
| |
| // It's a line comment if the start of the doc comment comes |
| // after other non-whitespace on the same line. |
| // ubyte** dc = (lineComment && anyToken) |
| // ? &t.lineComment |
| // : &t.blockComment; |
| |
| char[] dc = (lineComment && anyToken) ? t.lineComment : t.blockComment; |
| |
| // Combine with previous doc comment, if any |
| if (dc.length) { |
| dc = combineComments(dc, buf.toString().dup); |
| } |
| else { |
| dc = buf.toString().dup; |
| } |
| |
| // writefln( dc ); |
| |
| if (lineComment && anyToken) { |
| t.lineComment = dc; |
| } |
| else { |
| t.blockComment = dc; |
| } |
| } |
| } |
| |
| // character maps |
| static ubyte[256] cmtable; |
| |
| const int CMoctal = 0x1; |
| const int CMhex = 0x2; |
| const int CMidchar = 0x4; |
| |
| ubyte isoctal(ubyte c) { |
| return(cmtable[c] & CMoctal); |
| } |
| ubyte ishex(ubyte c) { |
| return(cmtable[c] & CMhex); |
| } |
| ubyte isidchar(ubyte c) { |
| return(cmtable[c] & CMidchar); |
| } |
| |
| static void cmtable_init() { |
| for (uint c = 0; c < cmtable.length; c++) { |
| if ('0' <= c && c <= '7') { |
| cmtable[c] |= CMoctal; |
| } |
| if (isdigit(c) || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F')) { |
| cmtable[c] |= CMhex; |
| } |
| if (isalnum(c) || c == '_') { |
| cmtable[c] |= CMidchar; |
| } |
| } |
| } |
| |
| |
| /+ |
| * struct StringValue |
| * { |
| * union |
| * { |
| * int intvalue; |
| * void *ptrvalue; |
| * dchar *string; |
| * } |
| * |
| * char[] lstring; |
| * } |
| * #define CASE_BASIC_TYPES |
| * case TOKwchar: case TOKdchar: |
| * case TOKbit: case TOKbool: case TOKchar: |
| * case TOKint8: case TOKuns8: |
| * case TOKint16: case TOKuns16: |
| * case TOKint32: case TOKuns32: |
| * case TOKint64: case TOKuns64: |
| * case TOKfloat32: case TOKfloat64: case TOKfloat80: |
| * case TOKimaginary32: case TOKimaginary64: case TOKimaginary80: |
| * case TOKcomplex32: case TOKcomplex64: case TOKcomplex80: |
| * case TOKvoid: |
| * |
| * #define CASE_BASIC_TYPES_X(t) \ |
| * case TOKvoid: t = Type::tvoid; goto LabelX; \ |
| * case TOKint8: t = Type::tint8; goto LabelX; \ |
| * case TOKuns8: t = Type::tuns8; goto LabelX; \ |
| * case TOKint16: t = Type::tint16; goto LabelX; \ |
| * case TOKuns16: t = Type::tuns16; goto LabelX; \ |
| * case TOKint32: t = Type::tint32; goto LabelX; \ |
| * case TOKuns32: t = Type::tuns32; goto LabelX; \ |
| * case TOKint64: t = Type::tint64; goto LabelX; \ |
| * case TOKuns64: t = Type::tuns64; goto LabelX; \ |
| * case TOKfloat32: t = Type::tfloat32; goto LabelX; \ |
| * case TOKfloat64: t = Type::tfloat64; goto LabelX; \ |
| * case TOKfloat80: t = Type::tfloat80; goto LabelX; \ |
| * case TOKimaginary32: t = Type::timaginary32; goto LabelX; \ |
| * case TOKimaginary64: t = Type::timaginary64; goto LabelX; \ |
| * case TOKimaginary80: t = Type::timaginary80; goto LabelX; \ |
| * case TOKcomplex32: t = Type::tcomplex32; goto LabelX; \ |
| * case TOKcomplex64: t = Type::tcomplex64; goto LabelX; \ |
| * case TOKcomplex80: t = Type::tcomplex80; goto LabelX; \ |
| * case TOKbit: t = Type::tbit; goto LabelX; \ |
| * case TOKchar: t = Type::tchar; goto LabelX; \ |
| * case TOKwchar: t = Type::twchar; goto LabelX; \ |
| * case TOKdchar: t = Type::tdchar; goto LabelX; \ |
| * LabelX |
| +/ |
