i2p/haiku-beta6
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Debugger: Minor refactoring.

Browse Source 
- Break the expression evaluator's tokenizer out into its own separate
  header so that the syntax highlighter can also make use of it.
This commit is contained in:
Rene Gollent
2014-11-29 21:28:08 -05:00
parent c04274bdd3
commit 4671959310
6 changed files with 526 additions and 428 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/apps/debugger/Jamfile
+1
View File
@@ -221,6 +221,7 @@ Application Debugger :
# source_language/expression_evaluators
CLanguageExpressionEvaluator.cpp
CLanguageTokenizer.cpp
# source_language/syntax_highlighters
CLanguageFamilySyntaxHighlighter.cpp
src/apps/debugger/source_language/CLanguageFamily.cpp
+3
View File
@@ -12,6 +12,7 @@
#include "CLanguageExpressionEvaluator.h"
#include "CLanguageFamilySyntaxHighlighter.h"
#include "CLanguageTokenizer.h"
#include "ExpressionInfo.h"
#include "TeamTypeInformation.h"
#include "StringValue.h"
@@ -19,6 +20,8 @@
#include "TypeLookupConstraints.h"
using CLanguage::ParseException;
CLanguageFamily::CLanguageFamily()
{
src/apps/debugger/source_language/expression_evaluators/CLanguageExpressionEvaluator.cpp
+6 -405
View File
@@ -13,13 +13,9 @@
#include <algorithm>
#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <strings.h>
#include "AutoLocker.h"
#include "CLanguageTokenizer.h"
#include "ExpressionInfo.h"
#include "FloatValue.h"
#include "IntegerValue.h"
@@ -33,40 +29,7 @@
#include "VariableValueNodeChild.h"
enum {
TOKEN_NONE = 0,
TOKEN_IDENTIFIER,
TOKEN_CONSTANT,
TOKEN_END_OF_LINE,
TOKEN_PLUS,
TOKEN_MINUS,
TOKEN_STAR,
TOKEN_SLASH,
TOKEN_MODULO,
TOKEN_POWER,
TOKEN_OPENING_BRACKET,
TOKEN_CLOSING_BRACKET,
TOKEN_LOGICAL_AND,
TOKEN_LOGICAL_OR,
TOKEN_LOGICAL_NOT,
TOKEN_BITWISE_AND,
TOKEN_BITWISE_OR,
TOKEN_BITWISE_NOT,
TOKEN_BITWISE_XOR,
TOKEN_EQ,
TOKEN_NE,
TOKEN_GT,
TOKEN_GE,
TOKEN_LT,
TOKEN_LE,
TOKEN_MEMBER_PTR
};
using namespace CLanguage;
enum operand_kind {
@@ -1425,369 +1388,6 @@ private:
};
// #pragma mark - CLanguageExpressionEvaluator::Token
struct CLanguageExpressionEvaluator::Token {
Token()
: string(""),
type(TOKEN_NONE),
value(0L),
position(0)
{
}
Token(const Token& other)
: string(other.string),
type(other.type),
value(other.value),
position(other.position)
{
}
Token(const char* string, int32 length, int32 position, int32 type)
: string(string, length),
type(type),
value(),
position(position)
{
}
Token& operator=(const Token& other)
{
string = other.string;
type = other.type;
value = other.value;
position = other.position;
return *this;
}
BString string;
int32 type;
BVariant value;
int32 position;
};
// #pragma mark - CLanguageExpressionEvaluator::Tokenizer
class CLanguageExpressionEvaluator::Tokenizer {
public:
Tokenizer()
: fString(""),
fCurrentChar(NULL),
fCurrentToken(),
fReuseToken(false)
{
}
void SetTo(const char* string)
{
fString = string;
fCurrentChar = fString.String();
fCurrentToken = Token();
fReuseToken = false;
}
const Token& NextToken()
{
if (fCurrentToken.type == TOKEN_END_OF_LINE)
return fCurrentToken;
if (fReuseToken) {
fReuseToken = false;
//printf("next token (recycled): '%s'\n", fCurrentToken.string.String());
return fCurrentToken;
}
while (*fCurrentChar != 0 && isspace(*fCurrentChar))
fCurrentChar++;
if (*fCurrentChar == 0)
return fCurrentToken = Token("", 0, _CurrentPos(), TOKEN_END_OF_LINE);
bool decimal = *fCurrentChar == '.' || *fCurrentChar == ',';
if (decimal || isdigit(*fCurrentChar)) {
if (*fCurrentChar == '0' && fCurrentChar[1] == 'x')
return _ParseHexOperand();
BString temp;
const char* begin = fCurrentChar;
// optional digits before the comma
while (isdigit(*fCurrentChar)) {
temp << *fCurrentChar;
fCurrentChar++;
}
// optional post comma part
// (required if there are no digits before the comma)
if (*fCurrentChar == '.' || *fCurrentChar == ',') {
decimal = true;
temp << '.';
fCurrentChar++;
// optional post comma digits
while (isdigit(*fCurrentChar)) {
temp << *fCurrentChar;
fCurrentChar++;
}
}
int32 length = fCurrentChar - begin;
if (length == 1 && decimal) {
// check for . operator
fCurrentChar = begin;
if (!_ParseOperator()) {
throw ParseException("unexpected character",
_CurrentPos());
}
return fCurrentToken;
}
BString test = temp;
test << "&_";
double value;
char t[2];
int32 matches = sscanf(test.String(), "%lf&%s", &value, t);
if (matches != 2) {
throw ParseException("error in constant",
_CurrentPos() - length);
}
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_CONSTANT);
if (decimal)
fCurrentToken.value.SetTo(value);
else {
fCurrentToken.value.SetTo((int64)strtoll(temp.String(), NULL,
10));
}
} else if (isalpha(*fCurrentChar)) {
const char* begin = fCurrentChar;
while (*fCurrentChar != 0 && (isalpha(*fCurrentChar)
|| isdigit(*fCurrentChar))) {
fCurrentChar++;
}
int32 length = fCurrentChar - begin;
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_IDENTIFIER);
} else {
if (!_ParseOperator()) {
int32 type = TOKEN_NONE;
switch (*fCurrentChar) {
case '\n':
type = TOKEN_END_OF_LINE;
break;
case '(':
type = TOKEN_OPENING_BRACKET;
break;
case ')':
type = TOKEN_CLOSING_BRACKET;
break;
case '\\':
case ':':
type = TOKEN_SLASH;
break;
default:
throw ParseException("unexpected character",
_CurrentPos());
}
fCurrentToken = Token(fCurrentChar, 1, _CurrentPos(), type);
fCurrentChar++;
}
}
return fCurrentToken;
}
bool _ParseOperator()
{
int32 type = TOKEN_NONE;
int32 length = 0;
switch (*fCurrentChar) {
case '+':
type = TOKEN_PLUS;
length = 1;
break;
case '-':
if (Peek() == '>') {
type = TOKEN_MEMBER_PTR;
length = 2;
} else {
type = TOKEN_MINUS;
length = 1;
}
break;
case '*':
if (Peek() == '*') {
type = TOKEN_POWER;
length = 2;
} else {
type = TOKEN_STAR;
length = 1;
}
break;
case '/':
type = TOKEN_SLASH;
length = 1;
break;
case '%':
type = TOKEN_MODULO;
length = 1;
break;
case '^':
type = TOKEN_BITWISE_XOR;
length = 1;
break;
case '&':
if (Peek() == '&') {
type = TOKEN_LOGICAL_AND;
length = 2;
} else {
type = TOKEN_BITWISE_AND;
length = 1;
}
break;
case '|':
if (Peek() == '|') {
type = TOKEN_LOGICAL_OR;
length = 2;
} else {
type = TOKEN_BITWISE_OR;
length = 1;
}
break;
case '!':
if (Peek() == '=') {
type = TOKEN_NE;
length = 2;
} else {
type = TOKEN_LOGICAL_NOT;
length = 1;
}
break;
case '=':
if (Peek() == '=') {
type = TOKEN_EQ;
length = 2;
}
break;
case '>':
if (Peek() == '=') {
type = TOKEN_GE;
length = 2;
} else {
type = TOKEN_GT;
length = 1;
}
break;
case '<':
if (Peek() == '=') {
type = TOKEN_LE;
length = 2;
} else {
type = TOKEN_LT;
length = 1;
}
break;
case '~':
type = TOKEN_BITWISE_NOT;
length = 1;
break;
case '.':
type = TOKEN_MEMBER_PTR;
length = 1;
break;
default:
break;
}
if (length == 0)
return false;
fCurrentToken = Token(fCurrentChar, length, _CurrentPos(), type);
fCurrentChar += length;
return true;
}
void RewindToken()
{
fReuseToken = true;
}
private:
char Peek() const
{
if (_CurrentPos() < fString.Length())
return *(fCurrentChar + 1);
return '\0';
}
static bool _IsHexDigit(char c)
{
return isdigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
}
Token& _ParseHexOperand()
{
const char* begin = fCurrentChar;
fCurrentChar += 2;
// skip "0x"
if (!_IsHexDigit(*fCurrentChar))
throw ParseException("expected hex digit", _CurrentPos());
fCurrentChar++;
while (_IsHexDigit(*fCurrentChar))
fCurrentChar++;
int32 length = fCurrentChar - begin;
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_CONSTANT);
fCurrentToken.value.SetTo((int64)strtoull(
fCurrentToken.string.String(), NULL, 16));
return fCurrentToken;
}
int32 _CurrentPos() const
{
return fCurrentChar - fString.String();
}
BString fString;
const char* fCurrentChar;
Token fCurrentToken;
bool fReuseToken;
};
// #pragma mark - CLanguageExpressionEvaluator
@@ -2191,15 +1791,16 @@ CLanguageExpressionEvaluator::_EatToken(int32 type)
break;
}
BString temp;
temp << "Expected " << expected.String() << " got '" << token.string << "'";
temp << "Expected " << expected.String() << " got '" << token.string
<< "'";
throw ParseException(temp.String(), token.position);
}
}
void
CLanguageExpressionEvaluator::_RequestValueIfNeeded(const Token& token,
ValueNodeChild* child)
CLanguageExpressionEvaluator::_RequestValueIfNeeded(
const Token& token, ValueNodeChild* child)
{
status_t state;
BString errorMessage;
src/apps/debugger/source_language/expression_evaluators/CLanguageExpressionEvaluator.h
+9 -23
View File
@@ -15,31 +15,18 @@
#include <String.h>
namespace CLanguage {
struct Token;
class Tokenizer;
}
class ValueNode;
class ValueNodeChild;
class ValueNodeManager;
class Variable;
class ParseException {
public:
ParseException(const char* message, int32 position)
: message(message),
position(position)
{
}
ParseException(const ParseException& other)
: message(other.message),
position(other.position)
{
}
BString message;
int32 position;
};
class ValueNeededException {
public:
ValueNeededException(ValueNode* node)
@@ -67,8 +54,6 @@ class CLanguageExpressionEvaluator {
private:
class Operand;
struct Token;
class Tokenizer;
private:
Operand _ParseSum();
@@ -80,10 +65,11 @@ class CLanguageExpressionEvaluator {
void _EatToken(int32 type);
void _RequestValueIfNeeded(const Token& token,
void _RequestValueIfNeeded(
const CLanguage::Token& token,
ValueNodeChild* child);
Tokenizer* fTokenizer;
CLanguage::Tokenizer* fTokenizer;
ValueNodeManager* fNodeManager;
};
src/apps/debugger/source_language/expression_evaluators/CLanguageTokenizer.cpp
+387
View File
@@ -0,0 +1,387 @@
/*
* Copyright 2006-2014 Haiku, Inc. All Rights Reserved.
* Distributed under the terms of the MIT License.
*
* Authors:
* Stephan Aßmus <superstippi@gmx.de>
* Rene Gollent <rene@gollent.com>
* John Scipione <jscipione@gmail.com>
* Ingo Weinhold <bonefish@cs.tu-berlin.de>
*/
#include "CLanguageTokenizer.h"
#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
using CLanguage::ParseException;
using CLanguage::Token;
using CLanguage::Tokenizer;
// #pragma mark - Token
Token::Token()
:
string(""),
type(TOKEN_NONE),
value(0L),
position(0)
{
}
Token::Token(const Token& other)
:
string(other.string),
type(other.type),
value(other.value),
position(other.position)
{
}
Token::Token(const char* string, int32 length, int32 position, int32 type)
:
string(string, length),
type(type),
value(),
position(position)
{
}
Token&
Token::operator=(const Token& other)
{
string = other.string;
type = other.type;
value = other.value;
position = other.position;
return *this;
}
// #pragma mark - Tokenizer
Tokenizer::Tokenizer()
:
fString(""),
fCurrentChar(NULL),
fCurrentToken(),
fReuseToken(false)
{
}
void
Tokenizer::SetTo(const char* string)
{
fString = string;
fCurrentChar = fString.String();
fCurrentToken = Token();
fReuseToken = false;
}
const Token&
Tokenizer::NextToken()
{
if (fCurrentToken.type == TOKEN_END_OF_LINE)
return fCurrentToken;
if (fReuseToken) {
fReuseToken = false;
return fCurrentToken;
}
while (*fCurrentChar != 0 && isspace(*fCurrentChar))
fCurrentChar++;
if (*fCurrentChar == 0) {
return fCurrentToken = Token("", 0, _CurrentPos(),
TOKEN_END_OF_LINE);
}
bool decimal = *fCurrentChar == '.' || *fCurrentChar == ',';
if (decimal || isdigit(*fCurrentChar)) {
if (*fCurrentChar == '0' && fCurrentChar[1] == 'x')
return _ParseHexOperand();
BString temp;
const char* begin = fCurrentChar;
// optional digits before the comma
while (isdigit(*fCurrentChar)) {
temp << *fCurrentChar;
fCurrentChar++;
}
// optional post comma part
// (required if there are no digits before the comma)
if (*fCurrentChar == '.' || *fCurrentChar == ',') {
decimal = true;
temp << '.';
fCurrentChar++;
// optional post comma digits
while (isdigit(*fCurrentChar)) {
temp << *fCurrentChar;
fCurrentChar++;
}
}
int32 length = fCurrentChar - begin;
if (length == 1 && decimal) {
// check for . operator
fCurrentChar = begin;
if (!_ParseOperator())
throw ParseException("unexpected character", _CurrentPos());
return fCurrentToken;
}
BString test = temp;
test << "&_";
double value;
char t[2];
int32 matches = sscanf(test.String(), "%lf&%s", &value, t);
if (matches != 2)
throw ParseException("error in constant", _CurrentPos() - length);
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_CONSTANT);
if (decimal)
fCurrentToken.value.SetTo(value);
else
fCurrentToken.value.SetTo((int64)strtoll(temp.String(), NULL, 10));
} else if (isalpha(*fCurrentChar)) {
const char* begin = fCurrentChar;
while (*fCurrentChar != 0 && (isalpha(*fCurrentChar)
|| isdigit(*fCurrentChar))) {
fCurrentChar++;
}
int32 length = fCurrentChar - begin;
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_IDENTIFIER);
} else {
if (!_ParseOperator()) {
int32 type = TOKEN_NONE;
switch (*fCurrentChar) {
case '\n':
type = TOKEN_END_OF_LINE;
break;
case '(':
type = TOKEN_OPENING_BRACKET;
break;
case ')':
type = TOKEN_CLOSING_BRACKET;
break;
case '\\':
case ':':
type = TOKEN_SLASH;
break;
default:
throw ParseException("unexpected character",
_CurrentPos());
}
fCurrentToken = Token(fCurrentChar, 1, _CurrentPos(),
type);
fCurrentChar++;
}
}
return fCurrentToken;
}
bool
Tokenizer::_ParseOperator()
{
int32 type = TOKEN_NONE;
int32 length = 0;
switch (*fCurrentChar) {
case '+':
type = TOKEN_PLUS;
length = 1;
break;
case '-':
if (_Peek() == '>') {
type = TOKEN_MEMBER_PTR;
length = 2;
} else {
type = TOKEN_MINUS;
length = 1;
}
break;
case '*':
if (_Peek() == '*') {
type = TOKEN_POWER;
length = 2;
} else {
type = TOKEN_STAR;
length = 1;
}
break;
case '/':
type = TOKEN_SLASH;
length = 1;
break;
case '%':
type = TOKEN_MODULO;
length = 1;
break;
case '^':
type = TOKEN_BITWISE_XOR;
length = 1;
break;
case '&':
if (_Peek() == '&') {
type = TOKEN_LOGICAL_AND;
length = 2;
} else {
type = TOKEN_BITWISE_AND;
length = 1;
}
break;
case '|':
if (_Peek() == '|') {
type = TOKEN_LOGICAL_OR;
length = 2;
} else {
type = TOKEN_BITWISE_OR;
length = 1;
}
break;
case '!':
if (_Peek() == '=') {
type = TOKEN_NE;
length = 2;
} else {
type = TOKEN_LOGICAL_NOT;
length = 1;
}
break;
case '=':
if (_Peek() == '=') {
type = TOKEN_EQ;
length = 2;
}
break;
case '>':
if (_Peek() == '=') {
type = TOKEN_GE;
length = 2;
} else {
type = TOKEN_GT;
length = 1;
}
break;
case '<':
if (_Peek() == '=') {
type = TOKEN_LE;
length = 2;
} else {
type = TOKEN_LT;
length = 1;
}
break;
case '~':
type = TOKEN_BITWISE_NOT;
length = 1;
break;
case '.':
type = TOKEN_MEMBER_PTR;
length = 1;
break;
default:
break;
}
if (length == 0)
return false;
fCurrentToken = Token(fCurrentChar, length, _CurrentPos(), type);
fCurrentChar += length;
return true;
}
void
Tokenizer::RewindToken()
{
fReuseToken = true;
}
char
Tokenizer::_Peek() const
{
if (_CurrentPos() < fString.Length())
return *(fCurrentChar + 1);
return '\0';
}
/*static*/ bool
Tokenizer::_IsHexDigit(char c)
{
return isdigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
}
Token&
Tokenizer::_ParseHexOperand()
{
const char* begin = fCurrentChar;
fCurrentChar += 2;
// skip "0x"
if (!_IsHexDigit(*fCurrentChar))
throw ParseException("expected hex digit", _CurrentPos());
fCurrentChar++;
while (_IsHexDigit(*fCurrentChar))
fCurrentChar++;
int32 length = fCurrentChar - begin;
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_CONSTANT);
fCurrentToken.value.SetTo((int64)strtoull(
fCurrentToken.string.String(), NULL, 16));
return fCurrentToken;
}
int32
Tokenizer::_CurrentPos() const
{
return fCurrentChar - fString.String();
}
src/apps/debugger/source_language/expression_evaluators/CLanguageTokenizer.h
+120
View File
@@ -0,0 +1,120 @@
/*
* Copyright 2006-2014 Haiku, Inc. All Rights Reserved.
* Distributed under the terms of the MIT License.
*
* Authors:
* Stephan Aßmus <superstippi@gmx.de>
* Rene Gollent <rene@gollent.com>
* John Scipione <jscipione@gmail.com>
* Ingo Weinhold <bonefish@cs.tu-berlin.de>
*/
#ifndef C_LANGUAGE_TOKENIZER
#define C_LANGUAGE_TOKENIZER
#include <String.h>
#include <Variant.h>
namespace CLanguage {
enum {
TOKEN_NONE = 0,
TOKEN_IDENTIFIER,
TOKEN_CONSTANT,
TOKEN_END_OF_LINE,
TOKEN_PLUS,
TOKEN_MINUS,
TOKEN_STAR,
TOKEN_SLASH,
TOKEN_MODULO,
TOKEN_POWER,
TOKEN_OPENING_BRACKET,
TOKEN_CLOSING_BRACKET,
TOKEN_LOGICAL_AND,
TOKEN_LOGICAL_OR,
TOKEN_LOGICAL_NOT,
TOKEN_BITWISE_AND,
TOKEN_BITWISE_OR,
TOKEN_BITWISE_NOT,
TOKEN_BITWISE_XOR,
TOKEN_EQ,
TOKEN_NE,
TOKEN_GT,
TOKEN_GE,
TOKEN_LT,
TOKEN_LE,
TOKEN_MEMBER_PTR
};
class ParseException {
public:
ParseException(const char* message, int32 position)
: message(message),
position(position)
{
}
ParseException(const ParseException& other)
: message(other.message),
position(other.position)
{
}
BString message;
int32 position;
};
struct Token {
Token();
Token(const Token& other);
Token(const char* string, int32 length,
int32 position, int32 type);
Token& operator=(const Token& other);
BString string;
int32 type;
BVariant value;
int32 position;
};
class Tokenizer {
public:
Tokenizer();
void SetTo(const char* string);
const Token& NextToken();
void RewindToken();
private:
bool _ParseOperator();
char _Peek() const;
static bool _IsHexDigit(char c);
Token& _ParseHexOperand();
int32 _CurrentPos() const;
private:
BString fString;
const char* fCurrentChar;
Token fCurrentToken;
bool fReuseToken;
};
} // namespace CLanguage
#endif // C_LANGUAGE_TOKENIZER