From 4671959310e053f0087f6f7a76fe7fba9639b0ae Mon Sep 17 00:00:00 2001 From: Rene Gollent Date: Wed, 26 Nov 2014 20:58:35 -0500 Subject: [PATCH] Debugger: Minor refactoring. - Break the expression evaluator's tokenizer out into its own separate header so that the syntax highlighter can also make use of it. --- src/apps/debugger/Jamfile | 1 + .../source_language/CLanguageFamily.cpp | 3 + .../CLanguageExpressionEvaluator.cpp | 411 +----------------- .../CLanguageExpressionEvaluator.h | 32 +- .../CLanguageTokenizer.cpp | 387 +++++++++++++++++ .../CLanguageTokenizer.h | 120 +++++ 6 files changed, 526 insertions(+), 428 deletions(-) create mode 100644 src/apps/debugger/source_language/expression_evaluators/CLanguageTokenizer.cpp create mode 100644 src/apps/debugger/source_language/expression_evaluators/CLanguageTokenizer.h diff --git a/src/apps/debugger/Jamfile b/src/apps/debugger/Jamfile index bf46073069..58ed75e187 100644 --- a/src/apps/debugger/Jamfile +++ b/src/apps/debugger/Jamfile @@ -221,6 +221,7 @@ Application Debugger : # source_language/expression_evaluators CLanguageExpressionEvaluator.cpp + CLanguageTokenizer.cpp # source_language/syntax_highlighters CLanguageFamilySyntaxHighlighter.cpp diff --git a/src/apps/debugger/source_language/CLanguageFamily.cpp b/src/apps/debugger/source_language/CLanguageFamily.cpp index cb75333de5..bbb516f2be 100644 --- a/src/apps/debugger/source_language/CLanguageFamily.cpp +++ b/src/apps/debugger/source_language/CLanguageFamily.cpp @@ -12,6 +12,7 @@ #include "CLanguageExpressionEvaluator.h" #include "CLanguageFamilySyntaxHighlighter.h" +#include "CLanguageTokenizer.h" #include "ExpressionInfo.h" #include "TeamTypeInformation.h" #include "StringValue.h" @@ -19,6 +20,8 @@ #include "TypeLookupConstraints.h" +using CLanguage::ParseException; + CLanguageFamily::CLanguageFamily() { diff --git a/src/apps/debugger/source_language/expression_evaluators/CLanguageExpressionEvaluator.cpp b/src/apps/debugger/source_language/expression_evaluators/CLanguageExpressionEvaluator.cpp index f4d506768c..f2de084d8e 100644 --- a/src/apps/debugger/source_language/expression_evaluators/CLanguageExpressionEvaluator.cpp +++ b/src/apps/debugger/source_language/expression_evaluators/CLanguageExpressionEvaluator.cpp @@ -13,13 +13,9 @@ #include -#include -#include -#include -#include - #include "AutoLocker.h" +#include "CLanguageTokenizer.h" #include "ExpressionInfo.h" #include "FloatValue.h" #include "IntegerValue.h" @@ -33,40 +29,7 @@ #include "VariableValueNodeChild.h" -enum { - TOKEN_NONE = 0, - TOKEN_IDENTIFIER, - TOKEN_CONSTANT, - TOKEN_END_OF_LINE, - - TOKEN_PLUS, - TOKEN_MINUS, - - TOKEN_STAR, - TOKEN_SLASH, - TOKEN_MODULO, - - TOKEN_POWER, - - TOKEN_OPENING_BRACKET, - TOKEN_CLOSING_BRACKET, - - TOKEN_LOGICAL_AND, - TOKEN_LOGICAL_OR, - TOKEN_LOGICAL_NOT, - TOKEN_BITWISE_AND, - TOKEN_BITWISE_OR, - TOKEN_BITWISE_NOT, - TOKEN_BITWISE_XOR, - TOKEN_EQ, - TOKEN_NE, - TOKEN_GT, - TOKEN_GE, - TOKEN_LT, - TOKEN_LE, - - TOKEN_MEMBER_PTR -}; +using namespace CLanguage; enum operand_kind { @@ -1425,369 +1388,6 @@ private: }; -// #pragma mark - CLanguageExpressionEvaluator::Token - - -struct CLanguageExpressionEvaluator::Token { - Token() - : string(""), - type(TOKEN_NONE), - value(0L), - position(0) - { - } - - Token(const Token& other) - : string(other.string), - type(other.type), - value(other.value), - position(other.position) - { - } - - Token(const char* string, int32 length, int32 position, int32 type) - : string(string, length), - type(type), - value(), - position(position) - { - } - - Token& operator=(const Token& other) - { - string = other.string; - type = other.type; - value = other.value; - position = other.position; - return *this; - } - - BString string; - int32 type; - BVariant value; - - int32 position; -}; - - -// #pragma mark - CLanguageExpressionEvaluator::Tokenizer - - -class CLanguageExpressionEvaluator::Tokenizer { -public: - Tokenizer() - : fString(""), - fCurrentChar(NULL), - fCurrentToken(), - fReuseToken(false) - { - } - - void SetTo(const char* string) - { - fString = string; - fCurrentChar = fString.String(); - fCurrentToken = Token(); - fReuseToken = false; - } - - const Token& NextToken() - { - if (fCurrentToken.type == TOKEN_END_OF_LINE) - return fCurrentToken; - - if (fReuseToken) { - fReuseToken = false; -//printf("next token (recycled): '%s'\n", fCurrentToken.string.String()); - return fCurrentToken; - } - - while (*fCurrentChar != 0 && isspace(*fCurrentChar)) - fCurrentChar++; - - if (*fCurrentChar == 0) - return fCurrentToken = Token("", 0, _CurrentPos(), TOKEN_END_OF_LINE); - - bool decimal = *fCurrentChar == '.' || *fCurrentChar == ','; - - if (decimal || isdigit(*fCurrentChar)) { - if (*fCurrentChar == '0' && fCurrentChar[1] == 'x') - return _ParseHexOperand(); - - BString temp; - - const char* begin = fCurrentChar; - - // optional digits before the comma - while (isdigit(*fCurrentChar)) { - temp << *fCurrentChar; - fCurrentChar++; - } - - // optional post comma part - // (required if there are no digits before the comma) - if (*fCurrentChar == '.' || *fCurrentChar == ',') { - decimal = true; - temp << '.'; - fCurrentChar++; - - // optional post comma digits - while (isdigit(*fCurrentChar)) { - temp << *fCurrentChar; - fCurrentChar++; - } - } - - int32 length = fCurrentChar - begin; - if (length == 1 && decimal) { - // check for . operator - fCurrentChar = begin; - if (!_ParseOperator()) { - throw ParseException("unexpected character", - _CurrentPos()); - } - - return fCurrentToken; - } - - BString test = temp; - test << "&_"; - double value; - char t[2]; - int32 matches = sscanf(test.String(), "%lf&%s", &value, t); - if (matches != 2) { - throw ParseException("error in constant", - _CurrentPos() - length); - } - - fCurrentToken = Token(begin, length, _CurrentPos() - length, - TOKEN_CONSTANT); - if (decimal) - fCurrentToken.value.SetTo(value); - else { - fCurrentToken.value.SetTo((int64)strtoll(temp.String(), NULL, - 10)); - } - - } else if (isalpha(*fCurrentChar)) { - const char* begin = fCurrentChar; - while (*fCurrentChar != 0 && (isalpha(*fCurrentChar) - || isdigit(*fCurrentChar))) { - fCurrentChar++; - } - int32 length = fCurrentChar - begin; - fCurrentToken = Token(begin, length, _CurrentPos() - length, - TOKEN_IDENTIFIER); - } else { - if (!_ParseOperator()) { - int32 type = TOKEN_NONE; - switch (*fCurrentChar) { - case '\n': - type = TOKEN_END_OF_LINE; - break; - - case '(': - type = TOKEN_OPENING_BRACKET; - break; - case ')': - type = TOKEN_CLOSING_BRACKET; - break; - - case '\\': - case ':': - type = TOKEN_SLASH; - break; - - default: - throw ParseException("unexpected character", - _CurrentPos()); - } - fCurrentToken = Token(fCurrentChar, 1, _CurrentPos(), type); - fCurrentChar++; - } - } - - return fCurrentToken; - } - - bool _ParseOperator() - { - int32 type = TOKEN_NONE; - int32 length = 0; - switch (*fCurrentChar) { - case '+': - type = TOKEN_PLUS; - length = 1; - break; - - case '-': - if (Peek() == '>') { - type = TOKEN_MEMBER_PTR; - length = 2; - } else { - type = TOKEN_MINUS; - length = 1; - } - break; - - case '*': - if (Peek() == '*') { - type = TOKEN_POWER; - length = 2; - } else { - type = TOKEN_STAR; - length = 1; - } - break; - - case '/': - type = TOKEN_SLASH; - length = 1; - break; - - case '%': - type = TOKEN_MODULO; - length = 1; - break; - - case '^': - type = TOKEN_BITWISE_XOR; - length = 1; - break; - - case '&': - if (Peek() == '&') { - type = TOKEN_LOGICAL_AND; - length = 2; - } else { - type = TOKEN_BITWISE_AND; - length = 1; - } - break; - - case '|': - if (Peek() == '|') { - type = TOKEN_LOGICAL_OR; - length = 2; - } else { - type = TOKEN_BITWISE_OR; - length = 1; - } - break; - - case '!': - if (Peek() == '=') { - type = TOKEN_NE; - length = 2; - } else { - type = TOKEN_LOGICAL_NOT; - length = 1; - } - break; - - case '=': - if (Peek() == '=') { - type = TOKEN_EQ; - length = 2; - } - break; - - case '>': - if (Peek() == '=') { - type = TOKEN_GE; - length = 2; - } else { - type = TOKEN_GT; - length = 1; - } - break; - - case '<': - if (Peek() == '=') { - type = TOKEN_LE; - length = 2; - } else { - type = TOKEN_LT; - length = 1; - } - break; - - case '~': - type = TOKEN_BITWISE_NOT; - length = 1; - break; - - case '.': - type = TOKEN_MEMBER_PTR; - length = 1; - break; - - default: - break; - } - - if (length == 0) - return false; - - fCurrentToken = Token(fCurrentChar, length, _CurrentPos(), type); - fCurrentChar += length; - - return true; - } - - void RewindToken() - { - fReuseToken = true; - } - - private: - char Peek() const - { - if (_CurrentPos() < fString.Length()) - return *(fCurrentChar + 1); - - return '\0'; - } - - static bool _IsHexDigit(char c) - { - return isdigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); - } - - Token& _ParseHexOperand() - { - const char* begin = fCurrentChar; - fCurrentChar += 2; - // skip "0x" - - if (!_IsHexDigit(*fCurrentChar)) - throw ParseException("expected hex digit", _CurrentPos()); - - fCurrentChar++; - while (_IsHexDigit(*fCurrentChar)) - fCurrentChar++; - - int32 length = fCurrentChar - begin; - fCurrentToken = Token(begin, length, _CurrentPos() - length, - TOKEN_CONSTANT); - - fCurrentToken.value.SetTo((int64)strtoull( - fCurrentToken.string.String(), NULL, 16)); - return fCurrentToken; - } - - int32 _CurrentPos() const - { - return fCurrentChar - fString.String(); - } - - BString fString; - const char* fCurrentChar; - Token fCurrentToken; - bool fReuseToken; -}; - - // #pragma mark - CLanguageExpressionEvaluator @@ -2191,15 +1791,16 @@ CLanguageExpressionEvaluator::_EatToken(int32 type) break; } BString temp; - temp << "Expected " << expected.String() << " got '" << token.string << "'"; + temp << "Expected " << expected.String() << " got '" << token.string + << "'"; throw ParseException(temp.String(), token.position); } } void -CLanguageExpressionEvaluator::_RequestValueIfNeeded(const Token& token, - ValueNodeChild* child) +CLanguageExpressionEvaluator::_RequestValueIfNeeded( + const Token& token, ValueNodeChild* child) { status_t state; BString errorMessage; diff --git a/src/apps/debugger/source_language/expression_evaluators/CLanguageExpressionEvaluator.h b/src/apps/debugger/source_language/expression_evaluators/CLanguageExpressionEvaluator.h index ef92daa34f..eaa6cf06c5 100644 --- a/src/apps/debugger/source_language/expression_evaluators/CLanguageExpressionEvaluator.h +++ b/src/apps/debugger/source_language/expression_evaluators/CLanguageExpressionEvaluator.h @@ -15,31 +15,18 @@ #include +namespace CLanguage { + struct Token; + class Tokenizer; +} + + class ValueNode; class ValueNodeChild; class ValueNodeManager; class Variable; -class ParseException { - public: - ParseException(const char* message, int32 position) - : message(message), - position(position) - { - } - - ParseException(const ParseException& other) - : message(other.message), - position(other.position) - { - } - - BString message; - int32 position; -}; - - class ValueNeededException { public: ValueNeededException(ValueNode* node) @@ -67,8 +54,6 @@ class CLanguageExpressionEvaluator { private: class Operand; - struct Token; - class Tokenizer; private: Operand _ParseSum(); @@ -80,10 +65,11 @@ class CLanguageExpressionEvaluator { void _EatToken(int32 type); - void _RequestValueIfNeeded(const Token& token, + void _RequestValueIfNeeded( + const CLanguage::Token& token, ValueNodeChild* child); - Tokenizer* fTokenizer; + CLanguage::Tokenizer* fTokenizer; ValueNodeManager* fNodeManager; }; diff --git a/src/apps/debugger/source_language/expression_evaluators/CLanguageTokenizer.cpp b/src/apps/debugger/source_language/expression_evaluators/CLanguageTokenizer.cpp new file mode 100644 index 0000000000..79b802cf6a --- /dev/null +++ b/src/apps/debugger/source_language/expression_evaluators/CLanguageTokenizer.cpp @@ -0,0 +1,387 @@ +/* + * Copyright 2006-2014 Haiku, Inc. All Rights Reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Stephan Aßmus + * Rene Gollent + * John Scipione + * Ingo Weinhold + */ + + +#include "CLanguageTokenizer.h" + +#include +#include +#include + + +using CLanguage::ParseException; +using CLanguage::Token; +using CLanguage::Tokenizer; + + +// #pragma mark - Token + + +Token::Token() + : + string(""), + type(TOKEN_NONE), + value(0L), + position(0) +{ +} + + +Token::Token(const Token& other) + : + string(other.string), + type(other.type), + value(other.value), + position(other.position) +{ +} + + +Token::Token(const char* string, int32 length, int32 position, int32 type) + : + string(string, length), + type(type), + value(), + position(position) +{ +} + + +Token& +Token::operator=(const Token& other) +{ + string = other.string; + type = other.type; + value = other.value; + position = other.position; + return *this; +} + + +// #pragma mark - Tokenizer + + +Tokenizer::Tokenizer() + : + fString(""), + fCurrentChar(NULL), + fCurrentToken(), + fReuseToken(false) +{ +} + + +void +Tokenizer::SetTo(const char* string) +{ + fString = string; + fCurrentChar = fString.String(); + fCurrentToken = Token(); + fReuseToken = false; +} + + +const Token& +Tokenizer::NextToken() +{ + if (fCurrentToken.type == TOKEN_END_OF_LINE) + return fCurrentToken; + + if (fReuseToken) { + fReuseToken = false; + return fCurrentToken; + } + + while (*fCurrentChar != 0 && isspace(*fCurrentChar)) + fCurrentChar++; + + if (*fCurrentChar == 0) { + return fCurrentToken = Token("", 0, _CurrentPos(), + TOKEN_END_OF_LINE); + } + + bool decimal = *fCurrentChar == '.' || *fCurrentChar == ','; + + if (decimal || isdigit(*fCurrentChar)) { + if (*fCurrentChar == '0' && fCurrentChar[1] == 'x') + return _ParseHexOperand(); + + BString temp; + + const char* begin = fCurrentChar; + + // optional digits before the comma + while (isdigit(*fCurrentChar)) { + temp << *fCurrentChar; + fCurrentChar++; + } + + // optional post comma part + // (required if there are no digits before the comma) + if (*fCurrentChar == '.' || *fCurrentChar == ',') { + decimal = true; + temp << '.'; + fCurrentChar++; + + // optional post comma digits + while (isdigit(*fCurrentChar)) { + temp << *fCurrentChar; + fCurrentChar++; + } + } + + int32 length = fCurrentChar - begin; + if (length == 1 && decimal) { + // check for . operator + fCurrentChar = begin; + if (!_ParseOperator()) + throw ParseException("unexpected character", _CurrentPos()); + + return fCurrentToken; + } + + BString test = temp; + test << "&_"; + double value; + char t[2]; + int32 matches = sscanf(test.String(), "%lf&%s", &value, t); + if (matches != 2) + throw ParseException("error in constant", _CurrentPos() - length); + + fCurrentToken = Token(begin, length, _CurrentPos() - length, + TOKEN_CONSTANT); + if (decimal) + fCurrentToken.value.SetTo(value); + else + fCurrentToken.value.SetTo((int64)strtoll(temp.String(), NULL, 10)); + } else if (isalpha(*fCurrentChar)) { + const char* begin = fCurrentChar; + while (*fCurrentChar != 0 && (isalpha(*fCurrentChar) + || isdigit(*fCurrentChar))) { + fCurrentChar++; + } + int32 length = fCurrentChar - begin; + fCurrentToken = Token(begin, length, _CurrentPos() - length, + TOKEN_IDENTIFIER); + } else { + if (!_ParseOperator()) { + int32 type = TOKEN_NONE; + switch (*fCurrentChar) { + case '\n': + type = TOKEN_END_OF_LINE; + break; + + case '(': + type = TOKEN_OPENING_BRACKET; + break; + case ')': + type = TOKEN_CLOSING_BRACKET; + break; + + case '\\': + case ':': + type = TOKEN_SLASH; + break; + + default: + throw ParseException("unexpected character", + _CurrentPos()); + } + fCurrentToken = Token(fCurrentChar, 1, _CurrentPos(), + type); + fCurrentChar++; + } + } + + return fCurrentToken; +} + + +bool +Tokenizer::_ParseOperator() +{ + int32 type = TOKEN_NONE; + int32 length = 0; + switch (*fCurrentChar) { + case '+': + type = TOKEN_PLUS; + length = 1; + break; + + case '-': + if (_Peek() == '>') { + type = TOKEN_MEMBER_PTR; + length = 2; + } else { + type = TOKEN_MINUS; + length = 1; + } + break; + + case '*': + if (_Peek() == '*') { + type = TOKEN_POWER; + length = 2; + } else { + type = TOKEN_STAR; + length = 1; + } + break; + + case '/': + type = TOKEN_SLASH; + length = 1; + break; + + case '%': + type = TOKEN_MODULO; + length = 1; + break; + + case '^': + type = TOKEN_BITWISE_XOR; + length = 1; + break; + + case '&': + if (_Peek() == '&') { + type = TOKEN_LOGICAL_AND; + length = 2; + } else { + type = TOKEN_BITWISE_AND; + length = 1; + } + break; + + case '|': + if (_Peek() == '|') { + type = TOKEN_LOGICAL_OR; + length = 2; + } else { + type = TOKEN_BITWISE_OR; + length = 1; + } + break; + + case '!': + if (_Peek() == '=') { + type = TOKEN_NE; + length = 2; + } else { + type = TOKEN_LOGICAL_NOT; + length = 1; + } + break; + + case '=': + if (_Peek() == '=') { + type = TOKEN_EQ; + length = 2; + } + break; + + case '>': + if (_Peek() == '=') { + type = TOKEN_GE; + length = 2; + } else { + type = TOKEN_GT; + length = 1; + } + break; + + case '<': + if (_Peek() == '=') { + type = TOKEN_LE; + length = 2; + } else { + type = TOKEN_LT; + length = 1; + } + break; + + case '~': + type = TOKEN_BITWISE_NOT; + length = 1; + break; + + case '.': + type = TOKEN_MEMBER_PTR; + length = 1; + break; + + default: + break; + } + + if (length == 0) + return false; + + fCurrentToken = Token(fCurrentChar, length, _CurrentPos(), type); + fCurrentChar += length; + + return true; +} + + +void +Tokenizer::RewindToken() +{ + fReuseToken = true; +} + + +char +Tokenizer::_Peek() const +{ + if (_CurrentPos() < fString.Length()) + return *(fCurrentChar + 1); + + return '\0'; +} + + +/*static*/ bool +Tokenizer::_IsHexDigit(char c) +{ + return isdigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + + +Token& +Tokenizer::_ParseHexOperand() +{ + const char* begin = fCurrentChar; + fCurrentChar += 2; + // skip "0x" + + if (!_IsHexDigit(*fCurrentChar)) + throw ParseException("expected hex digit", _CurrentPos()); + + fCurrentChar++; + while (_IsHexDigit(*fCurrentChar)) + fCurrentChar++; + + int32 length = fCurrentChar - begin; + fCurrentToken = Token(begin, length, _CurrentPos() - length, + TOKEN_CONSTANT); + + fCurrentToken.value.SetTo((int64)strtoull( + fCurrentToken.string.String(), NULL, 16)); + return fCurrentToken; +} + + +int32 +Tokenizer::_CurrentPos() const +{ + return fCurrentChar - fString.String(); +} diff --git a/src/apps/debugger/source_language/expression_evaluators/CLanguageTokenizer.h b/src/apps/debugger/source_language/expression_evaluators/CLanguageTokenizer.h new file mode 100644 index 0000000000..e38cae0cb4 --- /dev/null +++ b/src/apps/debugger/source_language/expression_evaluators/CLanguageTokenizer.h @@ -0,0 +1,120 @@ +/* + * Copyright 2006-2014 Haiku, Inc. All Rights Reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Stephan Aßmus + * Rene Gollent + * John Scipione + * Ingo Weinhold + */ +#ifndef C_LANGUAGE_TOKENIZER +#define C_LANGUAGE_TOKENIZER + + +#include + +#include + + +namespace CLanguage { + + +enum { + TOKEN_NONE = 0, + TOKEN_IDENTIFIER, + TOKEN_CONSTANT, + TOKEN_END_OF_LINE, + + TOKEN_PLUS, + TOKEN_MINUS, + + TOKEN_STAR, + TOKEN_SLASH, + TOKEN_MODULO, + + TOKEN_POWER, + + TOKEN_OPENING_BRACKET, + TOKEN_CLOSING_BRACKET, + + TOKEN_LOGICAL_AND, + TOKEN_LOGICAL_OR, + TOKEN_LOGICAL_NOT, + TOKEN_BITWISE_AND, + TOKEN_BITWISE_OR, + TOKEN_BITWISE_NOT, + TOKEN_BITWISE_XOR, + TOKEN_EQ, + TOKEN_NE, + TOKEN_GT, + TOKEN_GE, + TOKEN_LT, + TOKEN_LE, + + TOKEN_MEMBER_PTR +}; + + +class ParseException { + public: + ParseException(const char* message, int32 position) + : message(message), + position(position) + { + } + + ParseException(const ParseException& other) + : message(other.message), + position(other.position) + { + } + + BString message; + int32 position; +}; + + +struct Token { + Token(); + Token(const Token& other); + Token(const char* string, int32 length, + int32 position, int32 type); + Token& operator=(const Token& other); + + BString string; + int32 type; + BVariant value; + int32 position; +}; + + +class Tokenizer { +public: + Tokenizer(); + + void SetTo(const char* string); + + const Token& NextToken(); + void RewindToken(); +private: + bool _ParseOperator(); + char _Peek() const; + + static bool _IsHexDigit(char c); + + Token& _ParseHexOperand(); + int32 _CurrentPos() const; + +private: + BString fString; + const char* fCurrentChar; + Token fCurrentToken; + bool fReuseToken; +}; + + +} // namespace CLanguage + + +#endif // C_LANGUAGE_TOKENIZER