Commit 2cb5aac7 authored by oliver@apple.com's avatar oliver@apple.com

Tidy up lexer token ids

https://bugs.webkit.org/show_bug.cgi?id=42014

Reviewed by Darin Adler

Stop using character literals to identify single character tokens
and instead use symbolic names for all tokens.

* parser/ASTBuilder.h:
(JSC::ASTBuilder::makeBinaryNode):
* parser/JSParser.cpp:
(JSC::JSParser::consume):
(JSC::JSParser::match):
(JSC::JSParser::autoSemiColon):
(JSC::JSParser::JSParser):
(JSC::JSParser::parseProgram):
(JSC::JSParser::allowAutomaticSemicolon):
(JSC::JSParser::parseDoWhileStatement):
(JSC::JSParser::parseWhileStatement):
(JSC::JSParser::parseVarDeclarationList):
(JSC::JSParser::parseConstDeclarationList):
(JSC::JSParser::parseForStatement):
(JSC::JSParser::parseReturnStatement):
(JSC::JSParser::parseWithStatement):
(JSC::JSParser::parseSwitchStatement):
(JSC::JSParser::parseSwitchClauses):
(JSC::JSParser::parseSwitchDefaultClause):
(JSC::JSParser::parseTryStatement):
(JSC::JSParser::parseDebuggerStatement):
(JSC::JSParser::parseStatement):
(JSC::JSParser::parseFormalParameters):
(JSC::JSParser::parseFunctionInfo):
(JSC::JSParser::parseExpressionOrLabelStatement):
(JSC::JSParser::parseIfStatement):
(JSC::JSParser::parseExpression):
(JSC::JSParser::parseAssignmentExpression):
(JSC::JSParser::parseConditionalExpression):
(JSC::isUnaryOp):
(JSC::JSParser::isBinaryOperator):
(JSC::JSParser::parseBinaryExpression):
(JSC::JSParser::parseProperty):
(JSC::JSParser::parseObjectLiteral):
(JSC::JSParser::parseStrictObjectLiteral):
(JSC::JSParser::parseArrayLiteral):
(JSC::JSParser::parsePrimaryExpression):
(JSC::JSParser::parseArguments):
(JSC::JSParser::parseMemberExpression):
(JSC::JSParser::parseUnaryExpression):
* parser/JSParser.h:
(JSC::):
* parser/Lexer.cpp:
(JSC::):
(JSC::Lexer::lex):
* parser/Lexer.h:

git-svn-id: http://svn.webkit.org/repository/webkit/trunk@63024 268f45cc-cd09-0410-ab3c-d52691b4dbfc
parent 1baebe6a
2010-07-09 Oliver Hunt <oliver@apple.com>
Reviewed by Darin Adler.
Tidy up lexer token ids
https://bugs.webkit.org/show_bug.cgi?id=42014
Stop using character literals to identify single character tokens
and instead use symbolic names for all tokens.
* parser/ASTBuilder.h:
(JSC::ASTBuilder::makeBinaryNode):
* parser/JSParser.cpp:
(JSC::JSParser::consume):
(JSC::JSParser::match):
(JSC::JSParser::autoSemiColon):
(JSC::JSParser::JSParser):
(JSC::JSParser::parseProgram):
(JSC::JSParser::allowAutomaticSemicolon):
(JSC::JSParser::parseDoWhileStatement):
(JSC::JSParser::parseWhileStatement):
(JSC::JSParser::parseVarDeclarationList):
(JSC::JSParser::parseConstDeclarationList):
(JSC::JSParser::parseForStatement):
(JSC::JSParser::parseReturnStatement):
(JSC::JSParser::parseWithStatement):
(JSC::JSParser::parseSwitchStatement):
(JSC::JSParser::parseSwitchClauses):
(JSC::JSParser::parseSwitchDefaultClause):
(JSC::JSParser::parseTryStatement):
(JSC::JSParser::parseDebuggerStatement):
(JSC::JSParser::parseStatement):
(JSC::JSParser::parseFormalParameters):
(JSC::JSParser::parseFunctionInfo):
(JSC::JSParser::parseExpressionOrLabelStatement):
(JSC::JSParser::parseIfStatement):
(JSC::JSParser::parseExpression):
(JSC::JSParser::parseAssignmentExpression):
(JSC::JSParser::parseConditionalExpression):
(JSC::isUnaryOp):
(JSC::JSParser::isBinaryOperator):
(JSC::JSParser::parseBinaryExpression):
(JSC::JSParser::parseProperty):
(JSC::JSParser::parseObjectLiteral):
(JSC::JSParser::parseStrictObjectLiteral):
(JSC::JSParser::parseArrayLiteral):
(JSC::JSParser::parsePrimaryExpression):
(JSC::JSParser::parseArguments):
(JSC::JSParser::parseMemberExpression):
(JSC::JSParser::parseUnaryExpression):
* parser/JSParser.h:
(JSC::):
* parser/Lexer.cpp:
(JSC::):
(JSC::Lexer::lex):
* parser/Lexer.h:
2010-07-09 Gavin Barraclough <barraclough@apple.com>
Reviewed by Oliver Hunt.
......
......@@ -762,13 +762,13 @@ ExpressionNode* ASTBuilder::makeBinaryNode(int token, pair<ExpressionNode*, Bina
case AND:
return new (m_globalData) LogicalOpNode(m_globalData, lhs.first, rhs.first, OpLogicalAnd);
case '|':
case BITOR:
return new (m_globalData) BitOrNode(m_globalData, lhs.first, rhs.first, rhs.second.hasAssignment);
case '^':
case BITXOR:
return new (m_globalData) BitXOrNode(m_globalData, lhs.first, rhs.first, rhs.second.hasAssignment);
case '&':
case BITAND:
return new (m_globalData) BitAndNode(m_globalData, lhs.first, rhs.first, rhs.second.hasAssignment);
case EQEQ:
......@@ -783,10 +783,10 @@ ExpressionNode* ASTBuilder::makeBinaryNode(int token, pair<ExpressionNode*, Bina
case STRNEQ:
return new (m_globalData) NotStrictEqualNode(m_globalData, lhs.first, rhs.first, rhs.second.hasAssignment);
case '<':
case LT:
return new (m_globalData) LessNode(m_globalData, lhs.first, rhs.first, rhs.second.hasAssignment);
case '>':
case GT:
return new (m_globalData) GreaterNode(m_globalData, lhs.first, rhs.first, rhs.second.hasAssignment);
case LE:
......@@ -816,19 +816,19 @@ ExpressionNode* ASTBuilder::makeBinaryNode(int token, pair<ExpressionNode*, Bina
case URSHIFT:
return new (m_globalData) UnsignedRightShiftNode(m_globalData, lhs.first, rhs.first, rhs.second.hasAssignment);
case '+':
case PLUS:
return makeAddNode(lhs.first, rhs.first, rhs.second.hasAssignment);
case '-':
case MINUS:
return makeSubNode(lhs.first, rhs.first, rhs.second.hasAssignment);
case '*':
case TIMES:
return makeMultNode(lhs.first, rhs.first, rhs.second.hasAssignment);
case '/':
case DIVIDE:
return makeDivNode(lhs.first, rhs.first, rhs.second.hasAssignment);
case '%':
case MOD:
return new (m_globalData) ModNode(m_globalData, lhs.first, rhs.first, rhs.second.hasAssignment);
}
CRASH();
......
This diff is collapsed.
......@@ -33,69 +33,93 @@ class JSGlobalData;
class SourceCode;
enum JSTokenType {
NULLTOKEN = 258,
TRUETOKEN = 259,
FALSETOKEN = 260,
BREAK = 261,
CASE = 262,
DEFAULT = 263,
FOR = 264,
NEW = 265,
VAR = 266,
CONSTTOKEN = 267,
CONTINUE = 268,
FUNCTION = 269,
RETURN = 270,
VOIDTOKEN = 271,
DELETETOKEN = 272,
IF = 273,
THISTOKEN = 274,
DO = 275,
WHILE = 276,
INTOKEN = 277,
INSTANCEOF = 278,
TYPEOF = 279,
SWITCH = 280,
WITH = 281,
RESERVED = 282,
THROW = 283,
TRY = 284,
CATCH = 285,
FINALLY = 286,
DEBUGGER = 287,
IF_WITHOUT_ELSE = 288,
ELSE = 289,
EQEQ = 290,
NE = 291,
STREQ = 292,
STRNEQ = 293,
LE = 294,
GE = 295,
OR = 296,
AND = 297,
PLUSPLUS = 298,
MINUSMINUS = 299,
LSHIFT = 300,
RSHIFT = 301,
URSHIFT = 302,
PLUSEQUAL = 303,
MINUSEQUAL = 304,
MULTEQUAL = 305,
DIVEQUAL = 306,
LSHIFTEQUAL = 307,
RSHIFTEQUAL = 308,
URSHIFTEQUAL = 309,
ANDEQUAL = 310,
MODEQUAL = 311,
XOREQUAL = 312,
OREQUAL = 313,
OPENBRACE = 314,
CLOSEBRACE = 315,
NUMBER = 316,
IDENT = 317,
STRING = 318,
AUTOPLUSPLUS = 319,
AUTOMINUSMINUS = 320
NULLTOKEN,
TRUETOKEN,
FALSETOKEN,
BREAK,
CASE,
DEFAULT,
FOR,
NEW,
VAR,
CONSTTOKEN,
CONTINUE,
FUNCTION,
RETURN,
VOIDTOKEN,
DELETETOKEN,
IF,
THISTOKEN,
DO,
WHILE,
INTOKEN,
INSTANCEOF,
TYPEOF,
SWITCH,
WITH,
RESERVED,
THROW,
TRY,
CATCH,
FINALLY,
DEBUGGER,
IF_WITHOUT_ELSE,
ELSE,
EQUAL,
EQEQ,
NE,
STREQ,
STRNEQ,
LT,
GT,
LE,
GE,
OR,
AND,
PLUSPLUS,
MINUSMINUS,
LSHIFT,
RSHIFT,
URSHIFT,
PLUSEQUAL,
MINUSEQUAL,
MULTEQUAL,
DIVEQUAL,
LSHIFTEQUAL,
RSHIFTEQUAL,
URSHIFTEQUAL,
ANDEQUAL,
MODEQUAL,
XOREQUAL,
OREQUAL,
BITOR,
BITAND,
BITXOR,
PLUS,
MINUS,
TIMES,
DIVIDE,
MOD,
OPENBRACE,
CLOSEBRACE,
OPENPAREN,
CLOSEPAREN,
OPENBRACKET,
CLOSEBRACKET,
COMMA,
QUESTION,
TILDE,
EXCLAMATION,
NUMBER,
IDENT,
STRING,
AUTOPLUSPLUS,
AUTOMINUSMINUS,
SEMICOLON,
COLON,
DOT,
ERRORTOK,
EOFTOK
};
union JSTokenData {
......@@ -112,7 +136,7 @@ struct JSTokenInfo {
};
struct JSToken {
int m_type;
JSTokenType m_type;
JSTokenData m_data;
JSTokenInfo m_info;
};
......
......@@ -56,7 +56,14 @@ enum CharacterTypes {
CharacterLineTerminator,
CharacterExclamationMark,
CharacterSimple,
CharacterOpenParen,
CharacterCloseParen,
CharacterOpenBracket,
CharacterCloseBracket,
CharacterComma,
CharacterColon,
CharacterQuestion,
CharacterTilde,
CharacterQuote,
CharacterDot,
CharacterSlash,
......@@ -81,7 +88,7 @@ enum CharacterTypes {
};
// 128 ascii codes
static unsigned char AsciiCharacters[128] = {
static unsigned short AsciiCharacters[128] = {
/* 0 - Null */ CharacterInvalid,
/* 1 - Start of Heading */ CharacterInvalid,
/* 2 - Start of Text */ CharacterInvalid,
......@@ -122,11 +129,11 @@ static unsigned char AsciiCharacters[128] = {
/* 37 - % */ CharacterModulo,
/* 38 - & */ CharacterAnd,
/* 39 - ' */ CharacterQuote,
/* 40 - ( */ CharacterSimple,
/* 41 - ) */ CharacterSimple,
/* 40 - ( */ CharacterOpenParen,
/* 41 - ) */ CharacterCloseParen,
/* 42 - * */ CharacterMultiply,
/* 43 - + */ CharacterAdd,
/* 44 - , */ CharacterSimple,
/* 44 - , */ CharacterComma,
/* 45 - - */ CharacterSub,
/* 46 - . */ CharacterDot,
/* 47 - / */ CharacterSlash,
......@@ -140,12 +147,12 @@ static unsigned char AsciiCharacters[128] = {
/* 55 - 7 */ CharacterNumber,
/* 56 - 8 */ CharacterNumber,
/* 57 - 9 */ CharacterNumber,
/* 58 - : */ CharacterSimple,
/* 58 - : */ CharacterColon,
/* 59 - ; */ CharacterSemicolon,
/* 60 - < */ CharacterLess,
/* 61 - = */ CharacterEqual,
/* 62 - > */ CharacterGreater,
/* 63 - ? */ CharacterSimple,
/* 63 - ? */ CharacterQuestion,
/* 64 - @ */ CharacterInvalid,
/* 65 - A */ CharacterAlpha,
/* 66 - B */ CharacterAlpha,
......@@ -173,9 +180,9 @@ static unsigned char AsciiCharacters[128] = {
/* 88 - X */ CharacterAlpha,
/* 89 - Y */ CharacterAlpha,
/* 90 - Z */ CharacterAlpha,
/* 91 - [ */ CharacterSimple,
/* 91 - [ */ CharacterOpenBracket,
/* 92 - \ */ CharacterBackSlash,
/* 93 - ] */ CharacterSimple,
/* 93 - ] */ CharacterCloseBracket,
/* 94 - ^ */ CharacterXor,
/* 95 - _ */ CharacterAlpha,
/* 96 - ` */ CharacterInvalid,
......@@ -208,7 +215,7 @@ static unsigned char AsciiCharacters[128] = {
/* 123 - { */ CharacterOpenBrace,
/* 124 - | */ CharacterOr,
/* 125 - } */ CharacterCloseBrace,
/* 126 - ~ */ CharacterSimple,
/* 126 - ~ */ CharacterTilde,
/* 127 - Delete */ CharacterInvalid,
};
......@@ -465,13 +472,14 @@ ALWAYS_INLINE bool Lexer::parseString(JSTokenData* lvalp)
return true;
}
int Lexer::lex(JSTokenData* lvalp, JSTokenInfo* llocp)
JSTokenType Lexer::lex(JSTokenData* lvalp, JSTokenInfo* llocp)
{
ASSERT(!m_error);
ASSERT(m_buffer8.isEmpty());
ASSERT(m_buffer16.isEmpty());
int token = 0;
JSTokenType token = ERRORTOK;
int identChar = 0;
m_terminator = false;
start:
......@@ -481,7 +489,7 @@ start:
int startOffset = currentOffset();
if (UNLIKELY(m_current == -1))
return 0;
return EOFTOK;
m_delimited = false;
......@@ -516,7 +524,7 @@ start:
token = GE;
break;
}
token = '>';
token = GT;
break;
case CharacterEqual:
shift();
......@@ -530,7 +538,7 @@ start:
token = EQEQ;
break;
}
token = '=';
token = EQUAL;
break;
case CharacterLess:
shift();
......@@ -553,7 +561,7 @@ start:
token = LE;
break;
}
token = '<';
token = LT;
break;
case CharacterExclamationMark:
shift();
......@@ -567,7 +575,7 @@ start:
token = NE;
break;
}
token = '!';
token = EXCLAMATION;
break;
case CharacterAdd:
shift();
......@@ -581,7 +589,7 @@ start:
token = PLUSEQUAL;
break;
}
token = '+';
token = PLUS;
break;
case CharacterSub:
shift();
......@@ -599,7 +607,7 @@ start:
token = MINUSEQUAL;
break;
}
token = '-';
token = MINUS;
break;
case CharacterMultiply:
shift();
......@@ -608,7 +616,7 @@ start:
token = MULTEQUAL;
break;
}
token = '*';
token = TIMES;
break;
case CharacterSlash:
shift();
......@@ -625,7 +633,7 @@ start:
token = DIVEQUAL;
break;
}
token = '/';
token = DIVIDE;
break;
case CharacterAnd:
shift();
......@@ -639,7 +647,7 @@ start:
token = ANDEQUAL;
break;
}
token = '&';
token = BITAND;
break;
case CharacterXor:
shift();
......@@ -648,7 +656,7 @@ start:
token = XOREQUAL;
break;
}
token = '^';
token = BITXOR;
break;
case CharacterModulo:
shift();
......@@ -657,7 +665,7 @@ start:
token = MODEQUAL;
break;
}
token = '%';
token = MOD;
break;
case CharacterOr:
shift();
......@@ -671,7 +679,7 @@ start:
token = OR;
break;
}
token = '|';
token = BITOR;
break;
case CharacterDot:
shift();
......@@ -679,16 +687,44 @@ start:
record8('.');
goto inNumberAfterDecimalPoint;
}
token = '.';
token = DOT;
break;
case CharacterSimple:
token = m_current;
case CharacterOpenParen:
token = OPENPAREN;
shift();
break;
case CharacterCloseParen:
token = CLOSEPAREN;
shift();
break;
case CharacterOpenBracket:
token = OPENBRACKET;
shift();
break;
case CharacterCloseBracket:
token = CLOSEBRACKET;
shift();
break;
case CharacterComma:
token = COMMA;
shift();
break;
case CharacterColon:
token = COLON;
shift();
break;
case CharacterQuestion:
token = QUESTION;
shift();
break;
case CharacterTilde:
token = TILDE;
shift();
break;
case CharacterSemicolon:
m_delimited = true;
shift();
token = ';';
token = SEMICOLON;
break;
case CharacterOpenBrace:
lvalp->intValue = currentOffset();
......@@ -723,7 +759,7 @@ start:
m_atLineStart = true;
m_terminator = true;
if (lastTokenWasRestrKeyword()) {
token = ';';
token = SEMICOLON;
goto doneSemicolon;
}
goto start;
......@@ -758,10 +794,10 @@ startIdentifierWithBackslash: {
goto returnError;
shift();
token = getUnicodeCharacter();
if (UNLIKELY(token == -1))
identChar = getUnicodeCharacter();
if (UNLIKELY(identChar == -1))
goto returnError;
if (UNLIKELY(!isIdentStart(token)))
if (UNLIKELY(!isIdentStart(identChar)))
goto returnError;
goto inIdentifierAfterCharacterCheck;
}
......@@ -784,13 +820,13 @@ startIdentifierOrKeyword: {
if (UNLIKELY(m_current != 'u'))
goto returnError;
shift();
token = getUnicodeCharacter();
if (UNLIKELY(token == -1))
identChar = getUnicodeCharacter();
if (UNLIKELY(identChar == -1))
goto returnError;
if (UNLIKELY(!isIdentPart(token)))
if (UNLIKELY(!isIdentPart(identChar)))
goto returnError;
inIdentifierAfterCharacterCheck:
record16(token);
record16(identChar);
while (isIdentPart(m_current)) {
record16(m_current);
......@@ -802,7 +838,7 @@ inIdentifierAfterCharacterCheck:
inSingleLineComment:
while (!isLineTerminator(m_current)) {
if (UNLIKELY(m_current == -1))
return 0;
return EOFTOK;
shift();
}
shiftLineTerminator();
......@@ -969,7 +1005,7 @@ doneNumeric:
goto returnToken;
doneSemicolon:
token = ';';
token = SEMICOLON;
m_delimited = true;
goto returnToken;
......@@ -986,7 +1022,7 @@ doneIdentifierOrKeyword: {
m_delimited = false;
m_buffer16.resize(0);
const HashEntry* entry = m_keywordTable.entry(m_globalData, *lvalp->ident);
token = entry ? entry->lexerValue() : static_cast<int>(IDENT);
token = entry ? static_cast<JSTokenType>(entry->lexerValue()) : IDENT;
// Fall through into returnToken.
}
......@@ -1002,7 +1038,7 @@ returnToken: {
returnError:
m_error = true;
return -1;
return ERRORTOK;
}
bool Lexer::scanRegExp(const Identifier*& pattern, const Identifier*& flags, UChar patternPrefix)
......
......@@ -23,6 +23,7 @@
#ifndef Lexer_h
#define Lexer_h
#include "JSParser.h"
#include "Lookup.h"
#include "ParserArena.h"
#include "SourceCode.h"
......@@ -34,8 +35,6 @@
namespace JSC {
union JSTokenData;
struct JSTokenInfo;
class RegExp;
class Lexer : public Noncopyable {
......@@ -51,7 +50,7 @@ namespace JSC {
void setIsReparsing() { m_isReparsing = true; }
// Functions for the parser itself.
int lex(JSTokenData* lvalp, JSTokenInfo* llocp);
JSTokenType lex(JSTokenData* lvalp, JSTokenInfo* llocp);
int lineNumber() const { return m_lineNumber; }
void setLastLineNumber(int lastLineNumber) { m_lastLineNumber = lastLineNumber; }
int lastLineNumber() const { return m_lastLineNumber; }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment