Debugger: Minor refactoring.

- Break the expression evaluator's tokenizer out into its own separate
  header so that the syntax highlighter can also make use of it.

src/apps/debugger/Jamfile

@@ -221,6 +221,7 @@ Application Debugger :
 
 	# source_language/expression_evaluators
 	CLanguageExpressionEvaluator.cpp
+	CLanguageTokenizer.cpp
 
 	# source_language/syntax_highlighters
 	CLanguageFamilySyntaxHighlighter.cpp

src/apps/debugger/source_language/CLanguageFamily.cpp

@@ -12,6 +12,7 @@
 
 #include "CLanguageExpressionEvaluator.h"
 #include "CLanguageFamilySyntaxHighlighter.h"
+#include "CLanguageTokenizer.h"
 #include "ExpressionInfo.h"
 #include "TeamTypeInformation.h"
 #include "StringValue.h"
@@ -19,6 +20,8 @@
 #include "TypeLookupConstraints.h"
 
 
+using CLanguage::ParseException;
+
 
 CLanguageFamily::CLanguageFamily()
 {

src/apps/debugger/source_language/expression_evaluators/CLanguageExpressionEvaluator.cpp

@@ -13,13 +13,9 @@
 
 #include <algorithm>
 
-#include <ctype.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-
 #include "AutoLocker.h"
 
+#include "CLanguageTokenizer.h"
 #include "ExpressionInfo.h"
 #include "FloatValue.h"
 #include "IntegerValue.h"
@@ -33,40 +29,7 @@
 #include "VariableValueNodeChild.h"
 
 
-enum {
+using namespace CLanguage;
-	TOKEN_NONE					= 0,
-	TOKEN_IDENTIFIER,
-	TOKEN_CONSTANT,
-	TOKEN_END_OF_LINE,
-
-	TOKEN_PLUS,
-	TOKEN_MINUS,
-
-	TOKEN_STAR,
-	TOKEN_SLASH,
-	TOKEN_MODULO,
-
-	TOKEN_POWER,
-
-	TOKEN_OPENING_BRACKET,
-	TOKEN_CLOSING_BRACKET,
-
-	TOKEN_LOGICAL_AND,
-	TOKEN_LOGICAL_OR,
-	TOKEN_LOGICAL_NOT,
-	TOKEN_BITWISE_AND,
-	TOKEN_BITWISE_OR,
-	TOKEN_BITWISE_NOT,
-	TOKEN_BITWISE_XOR,
-	TOKEN_EQ,
-	TOKEN_NE,
-	TOKEN_GT,
-	TOKEN_GE,
-	TOKEN_LT,
-	TOKEN_LE,
-
-	TOKEN_MEMBER_PTR
-};
 
 
 enum operand_kind {
@@ -1425,369 +1388,6 @@ private:
 };
 
 
-// #pragma mark - CLanguageExpressionEvaluator::Token
-
-
-struct CLanguageExpressionEvaluator::Token {
-	Token()
-		: string(""),
-		  type(TOKEN_NONE),
-		  value(0L),
-		  position(0)
-	{
-	}
-
-	Token(const Token& other)
-		: string(other.string),
-		  type(other.type),
-		  value(other.value),
-		  position(other.position)
-	{
-	}
-
-	Token(const char* string, int32 length, int32 position, int32 type)
-		: string(string, length),
-		  type(type),
-		  value(),
-		  position(position)
-	{
-	}
-
-	Token& operator=(const Token& other)
-	{
-		string = other.string;
-		type = other.type;
-		value = other.value;
-		position = other.position;
-		return *this;
-	}
-
-	BString		string;
-	int32		type;
-	BVariant	value;
-
-	int32		position;
-};
-
-
-// #pragma mark - CLanguageExpressionEvaluator::Tokenizer
-
-
-class CLanguageExpressionEvaluator::Tokenizer {
-public:
-	Tokenizer()
-		: fString(""),
-		  fCurrentChar(NULL),
-		  fCurrentToken(),
-		  fReuseToken(false)
-	{
-	}
-
-	void SetTo(const char* string)
-	{
-		fString = string;
-		fCurrentChar = fString.String();
-		fCurrentToken = Token();
-		fReuseToken = false;
-	}
-
-	const Token& NextToken()
-	{
-		if (fCurrentToken.type == TOKEN_END_OF_LINE)
-			return fCurrentToken;
-
-		if (fReuseToken) {
-			fReuseToken = false;
-//printf("next token (recycled): '%s'\n", fCurrentToken.string.String());
-			return fCurrentToken;
-		}
-
-		while (*fCurrentChar != 0 && isspace(*fCurrentChar))
-			fCurrentChar++;
-
-		if (*fCurrentChar == 0)
-			return fCurrentToken = Token("", 0, _CurrentPos(), TOKEN_END_OF_LINE);
-
-		bool decimal = *fCurrentChar == '.' || *fCurrentChar == ',';
-
-		if (decimal || isdigit(*fCurrentChar)) {
-			if (*fCurrentChar == '0' && fCurrentChar[1] == 'x')
-				return _ParseHexOperand();
-
-			BString temp;
-
-			const char* begin = fCurrentChar;
-
-			// optional digits before the comma
-			while (isdigit(*fCurrentChar)) {
-				temp << *fCurrentChar;
-				fCurrentChar++;
-			}
-
-			// optional post comma part
-			// (required if there are no digits before the comma)
-			if (*fCurrentChar == '.' || *fCurrentChar == ',') {
-				decimal = true;
-				temp << '.';
-				fCurrentChar++;
-
-				// optional post comma digits
-				while (isdigit(*fCurrentChar)) {
-					temp << *fCurrentChar;
-					fCurrentChar++;
-				}
-			}
-
-			int32 length = fCurrentChar - begin;
-			if (length == 1 && decimal) {
-				// check for . operator
-				fCurrentChar = begin;
-				if (!_ParseOperator()) {
-					throw ParseException("unexpected character",
-						_CurrentPos());
-				}
-
-				return fCurrentToken;
-			}
-
-			BString test = temp;
-			test << "&_";
-			double value;
-			char t[2];
-			int32 matches = sscanf(test.String(), "%lf&%s", &value, t);
-			if (matches != 2) {
-				throw ParseException("error in constant",
-					_CurrentPos() - length);
-			}
-
-			fCurrentToken = Token(begin, length, _CurrentPos() - length,
-				TOKEN_CONSTANT);
-			if (decimal)
-				fCurrentToken.value.SetTo(value);
-			else {
-				fCurrentToken.value.SetTo((int64)strtoll(temp.String(), NULL,
-						10));
-			}
-
-		} else if (isalpha(*fCurrentChar)) {
-			const char* begin = fCurrentChar;
-			while (*fCurrentChar != 0 && (isalpha(*fCurrentChar)
-				|| isdigit(*fCurrentChar))) {
-				fCurrentChar++;
-			}
-			int32 length = fCurrentChar - begin;
-			fCurrentToken = Token(begin, length, _CurrentPos() - length,
-				TOKEN_IDENTIFIER);
-		} else {
-			if (!_ParseOperator()) {
-				int32 type = TOKEN_NONE;
-				switch (*fCurrentChar) {
-					case '\n':
-						type = TOKEN_END_OF_LINE;
-						break;
-
-					case '(':
-						type = TOKEN_OPENING_BRACKET;
-						break;
-					case ')':
-						type = TOKEN_CLOSING_BRACKET;
-						break;
-
-					case '\\':
-					case ':':
-						type = TOKEN_SLASH;
-						break;
-
-					default:
-						throw ParseException("unexpected character",
-							_CurrentPos());
-				}
-				fCurrentToken = Token(fCurrentChar, 1, _CurrentPos(), type);
-				fCurrentChar++;
-			}
-		}
-
-		return fCurrentToken;
-	}
-
-	bool _ParseOperator()
-	{
-		int32 type = TOKEN_NONE;
-		int32 length = 0;
-		switch (*fCurrentChar) {
-			case '+':
-				type = TOKEN_PLUS;
-				length = 1;
-				break;
-
-			case '-':
-				 if (Peek() == '>') {
-				 	type = TOKEN_MEMBER_PTR;
-				 	length = 2;
-				 } else {
-					type = TOKEN_MINUS;
-					length = 1;
-				 }
-				break;
-
-			case '*':
-				if (Peek() == '*')  {
-					type = TOKEN_POWER;
-					length = 2;
-				} else {
-					type = TOKEN_STAR;
-					length = 1;
-				}
-				break;
-
-			case '/':
-				type = TOKEN_SLASH;
-				length = 1;
-				break;
-
-			case '%':
-				type = TOKEN_MODULO;
-				length = 1;
-				break;
-
-			case '^':
-				type = TOKEN_BITWISE_XOR;
-				length = 1;
-				break;
-
-			case '&':
-				if (Peek() == '&') {
-				 	type = TOKEN_LOGICAL_AND;
-				 	length = 2;
-				} else {
-					type = TOKEN_BITWISE_AND;
-					length = 1;
-				}
-				break;
-
-			case '|':
-				if (Peek() == '|') {
-					type = TOKEN_LOGICAL_OR;
-					length = 2;
-				} else {
-					type = TOKEN_BITWISE_OR;
-					length = 1;
-				}
-				break;
-
-			case '!':
-				if (Peek() == '=') {
-					type = TOKEN_NE;
-					length = 2;
-				} else {
-					type = TOKEN_LOGICAL_NOT;
-					length = 1;
-				}
-				break;
-
-			case '=':
-				if (Peek() == '=') {
-					type = TOKEN_EQ;
-					length = 2;
-				}
-				break;
-
-			case '>':
-				if (Peek() == '=') {
-					type = TOKEN_GE;
-					length = 2;
-				} else {
-					type = TOKEN_GT;
-					length = 1;
-				}
-				break;
-
-			case '<':
-				if (Peek() == '=') {
-					type = TOKEN_LE;
-					length = 2;
-				} else {
-					type = TOKEN_LT;
-					length = 1;
-				}
-				break;
-
-			case '~':
-				type = TOKEN_BITWISE_NOT;
-				length = 1;
-				break;
-
-			case '.':
-				type = TOKEN_MEMBER_PTR;
-				length = 1;
-				break;
-
-			default:
-				break;
-		}
-
-		if (length == 0)
-			return false;
-
-		fCurrentToken = Token(fCurrentChar, length, _CurrentPos(), type);
-		fCurrentChar += length;
-
-		return true;
-	}
-
-	void RewindToken()
-	{
-		fReuseToken = true;
-	}
-
- private:
- 	char Peek() const
- 	{
- 		if (_CurrentPos() < fString.Length())
- 			return *(fCurrentChar + 1);
-
- 		return '\0';
- 	}
-
-	static bool _IsHexDigit(char c)
-	{
-		return isdigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
-	}
-
-	Token& _ParseHexOperand()
-	{
-		const char* begin = fCurrentChar;
-		fCurrentChar += 2;
-			// skip "0x"
-
-		if (!_IsHexDigit(*fCurrentChar))
-			throw ParseException("expected hex digit", _CurrentPos());
-
-		fCurrentChar++;
-		while (_IsHexDigit(*fCurrentChar))
-			fCurrentChar++;
-
-		int32 length = fCurrentChar - begin;
-		fCurrentToken = Token(begin, length, _CurrentPos() - length,
-			TOKEN_CONSTANT);
-
-		fCurrentToken.value.SetTo((int64)strtoull(
-				fCurrentToken.string.String(), NULL, 16));
-		return fCurrentToken;
-	}
-
-	int32 _CurrentPos() const
-	{
-		return fCurrentChar - fString.String();
-	}
-
-	BString		fString;
-	const char*	fCurrentChar;
-	Token		fCurrentToken;
-	bool		fReuseToken;
-};
-
-
 // #pragma mark - CLanguageExpressionEvaluator
 
 
@@ -2191,15 +1791,16 @@ CLanguageExpressionEvaluator::_EatToken(int32 type)
 				break;
 		}
 		BString temp;
-		temp << "Expected " << expected.String() << " got '" << token.string << "'";
+		temp << "Expected " << expected.String() << " got '" << token.string
+			<< "'";
 		throw ParseException(temp.String(), token.position);
 	}
 }
 
 
 void
-CLanguageExpressionEvaluator::_RequestValueIfNeeded(const Token& token,
+CLanguageExpressionEvaluator::_RequestValueIfNeeded(
-	ValueNodeChild* child)
+	const Token& token, ValueNodeChild* child)
 {
 	status_t state;
 	BString errorMessage;

src/apps/debugger/source_language/expression_evaluators/CLanguageExpressionEvaluator.h

@@ -15,31 +15,18 @@
 #include <String.h>
 
 
+namespace CLanguage {
+	struct Token;
+	class Tokenizer;
+}
+
+
 class ValueNode;
 class ValueNodeChild;
 class ValueNodeManager;
 class Variable;
 
 
-class ParseException {
- public:
-	ParseException(const char* message, int32 position)
-		: message(message),
-		  position(position)
-	{
-	}
-
-	ParseException(const ParseException& other)
-		: message(other.message),
-		  position(other.position)
-	{
-	}
-
-	BString	message;
-	int32	position;
-};
-
-
 class ValueNeededException {
 public:
 	ValueNeededException(ValueNode* node)
@@ -67,8 +54,6 @@ class CLanguageExpressionEvaluator {
 
  private:
 			class Operand;
-			struct Token;
-			class Tokenizer;
 
  private:
 			Operand				_ParseSum();
@@ -80,10 +65,11 @@ class CLanguageExpressionEvaluator {
 
 			void				_EatToken(int32 type);
 
-			void				_RequestValueIfNeeded(const Token& token,
+			void				_RequestValueIfNeeded(
+									const CLanguage::Token& token,
 									ValueNodeChild* child);
 
-			Tokenizer*			fTokenizer;
+			CLanguage::Tokenizer* fTokenizer;
 			ValueNodeManager*	fNodeManager;
 };
 

src/apps/debugger/source_language/expression_evaluators/CLanguageTokenizer.cpp

@@ -0,0 +1,387 @@
+/*
+ * Copyright 2006-2014 Haiku, Inc. All Rights Reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Stephan Aßmus <[email protected]>
+ *		Rene Gollent <[email protected]>
+ *		John Scipione <[email protected]>
+ *		Ingo Weinhold <[email protected]>
+ */
+
+
+#include "CLanguageTokenizer.h"
+
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+
+using CLanguage::ParseException;
+using CLanguage::Token;
+using CLanguage::Tokenizer;
+
+
+// #pragma mark - Token
+
+
+Token::Token()
+	:
+	string(""),
+	type(TOKEN_NONE),
+	value(0L),
+	position(0)
+{
+}
+
+
+Token::Token(const Token& other)
+	:
+	string(other.string),
+	type(other.type),
+	value(other.value),
+	position(other.position)
+{
+}
+
+
+Token::Token(const char* string, int32 length, int32 position, int32 type)
+	:
+	string(string, length),
+	type(type),
+	value(),
+	position(position)
+{
+}
+
+
+Token&
+Token::operator=(const Token& other)
+{
+	string = other.string;
+	type = other.type;
+	value = other.value;
+	position = other.position;
+	return *this;
+}
+
+
+// #pragma mark - Tokenizer
+
+
+Tokenizer::Tokenizer()
+	:
+	fString(""),
+	fCurrentChar(NULL),
+	fCurrentToken(),
+	fReuseToken(false)
+{
+}
+
+
+void
+Tokenizer::SetTo(const char* string)
+{
+	fString = string;
+	fCurrentChar = fString.String();
+	fCurrentToken = Token();
+	fReuseToken = false;
+}
+
+
+const Token&
+Tokenizer::NextToken()
+{
+	if (fCurrentToken.type == TOKEN_END_OF_LINE)
+		return fCurrentToken;
+
+	if (fReuseToken) {
+		fReuseToken = false;
+		return fCurrentToken;
+	}
+
+	while (*fCurrentChar != 0 && isspace(*fCurrentChar))
+		fCurrentChar++;
+
+	if (*fCurrentChar == 0) {
+		return fCurrentToken = Token("", 0, _CurrentPos(),
+			TOKEN_END_OF_LINE);
+	}
+
+	bool decimal = *fCurrentChar == '.' || *fCurrentChar == ',';
+
+	if (decimal || isdigit(*fCurrentChar)) {
+		if (*fCurrentChar == '0' && fCurrentChar[1] == 'x')
+			return _ParseHexOperand();
+
+		BString temp;
+
+		const char* begin = fCurrentChar;
+
+		// optional digits before the comma
+		while (isdigit(*fCurrentChar)) {
+			temp << *fCurrentChar;
+			fCurrentChar++;
+		}
+
+		// optional post comma part
+		// (required if there are no digits before the comma)
+		if (*fCurrentChar == '.' || *fCurrentChar == ',') {
+			decimal = true;
+			temp << '.';
+			fCurrentChar++;
+
+			// optional post comma digits
+			while (isdigit(*fCurrentChar)) {
+				temp << *fCurrentChar;
+				fCurrentChar++;
+			}
+		}
+
+		int32 length = fCurrentChar - begin;
+		if (length == 1 && decimal) {
+			// check for . operator
+			fCurrentChar = begin;
+			if (!_ParseOperator())
+				throw ParseException("unexpected character", _CurrentPos());
+
+			return fCurrentToken;
+		}
+
+		BString test = temp;
+		test << "&_";
+		double value;
+		char t[2];
+		int32 matches = sscanf(test.String(), "%lf&%s", &value, t);
+		if (matches != 2)
+			throw ParseException("error in constant", _CurrentPos() - length);
+
+		fCurrentToken = Token(begin, length, _CurrentPos() - length,
+			TOKEN_CONSTANT);
+		if (decimal)
+			fCurrentToken.value.SetTo(value);
+		else
+			fCurrentToken.value.SetTo((int64)strtoll(temp.String(), NULL, 10));
+	} else if (isalpha(*fCurrentChar)) {
+		const char* begin = fCurrentChar;
+		while (*fCurrentChar != 0 && (isalpha(*fCurrentChar)
+			|| isdigit(*fCurrentChar))) {
+			fCurrentChar++;
+		}
+		int32 length = fCurrentChar - begin;
+		fCurrentToken = Token(begin, length, _CurrentPos() - length,
+			TOKEN_IDENTIFIER);
+	} else {
+		if (!_ParseOperator()) {
+			int32 type = TOKEN_NONE;
+			switch (*fCurrentChar) {
+				case '\n':
+					type = TOKEN_END_OF_LINE;
+					break;
+
+				case '(':
+					type = TOKEN_OPENING_BRACKET;
+					break;
+				case ')':
+					type = TOKEN_CLOSING_BRACKET;
+					break;
+
+				case '\\':
+				case ':':
+					type = TOKEN_SLASH;
+					break;
+
+				default:
+					throw ParseException("unexpected character",
+						_CurrentPos());
+			}
+			fCurrentToken = Token(fCurrentChar, 1, _CurrentPos(),
+				type);
+			fCurrentChar++;
+		}
+	}
+
+	return fCurrentToken;
+}
+
+
+bool
+Tokenizer::_ParseOperator()
+{
+	int32 type = TOKEN_NONE;
+	int32 length = 0;
+	switch (*fCurrentChar) {
+		case '+':
+			type = TOKEN_PLUS;
+			length = 1;
+			break;
+
+		case '-':
+			 if (_Peek() == '>') {
+			 	type = TOKEN_MEMBER_PTR;
+			 	length = 2;
+			 } else {
+				type = TOKEN_MINUS;
+				length = 1;
+			 }
+			break;
+
+		case '*':
+			if (_Peek() == '*')  {
+				type = TOKEN_POWER;
+				length = 2;
+			} else {
+				type = TOKEN_STAR;
+				length = 1;
+			}
+			break;
+
+		case '/':
+			type = TOKEN_SLASH;
+			length = 1;
+			break;
+
+		case '%':
+			type = TOKEN_MODULO;
+			length = 1;
+			break;
+
+		case '^':
+			type = TOKEN_BITWISE_XOR;
+			length = 1;
+			break;
+
+		case '&':
+			if (_Peek() == '&') {
+			 	type = TOKEN_LOGICAL_AND;
+			 	length = 2;
+			} else {
+				type = TOKEN_BITWISE_AND;
+				length = 1;
+			}
+			break;
+
+		case '|':
+			if (_Peek() == '|') {
+				type = TOKEN_LOGICAL_OR;
+				length = 2;
+			} else {
+				type = TOKEN_BITWISE_OR;
+				length = 1;
+			}
+			break;
+
+		case '!':
+			if (_Peek() == '=') {
+				type = TOKEN_NE;
+				length = 2;
+			} else {
+				type = TOKEN_LOGICAL_NOT;
+				length = 1;
+			}
+			break;
+
+		case '=':
+			if (_Peek() == '=') {
+				type = TOKEN_EQ;
+				length = 2;
+			}
+			break;
+
+		case '>':
+			if (_Peek() == '=') {
+				type = TOKEN_GE;
+				length = 2;
+			} else {
+				type = TOKEN_GT;
+				length = 1;
+			}
+			break;
+
+		case '<':
+			if (_Peek() == '=') {
+				type = TOKEN_LE;
+				length = 2;
+			} else {
+				type = TOKEN_LT;
+				length = 1;
+			}
+			break;
+
+		case '~':
+			type = TOKEN_BITWISE_NOT;
+			length = 1;
+			break;
+
+		case '.':
+			type = TOKEN_MEMBER_PTR;
+			length = 1;
+			break;
+
+		default:
+			break;
+	}
+
+	if (length == 0)
+		return false;
+
+	fCurrentToken = Token(fCurrentChar, length, _CurrentPos(), type);
+	fCurrentChar += length;
+
+	return true;
+}
+
+
+void
+Tokenizer::RewindToken()
+{
+	fReuseToken = true;
+}
+
+
+char
+Tokenizer::_Peek() const
+{
+	if (_CurrentPos() < fString.Length())
+		return *(fCurrentChar + 1);
+
+	return '\0';
+}
+
+
+/*static*/ bool
+Tokenizer::_IsHexDigit(char c)
+{
+	return isdigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+}
+
+
+Token&
+Tokenizer::_ParseHexOperand()
+{
+	const char* begin = fCurrentChar;
+	fCurrentChar += 2;
+		// skip "0x"
+
+	if (!_IsHexDigit(*fCurrentChar))
+		throw ParseException("expected hex digit", _CurrentPos());
+
+	fCurrentChar++;
+	while (_IsHexDigit(*fCurrentChar))
+		fCurrentChar++;
+
+	int32 length = fCurrentChar - begin;
+	fCurrentToken = Token(begin, length, _CurrentPos() - length,
+		TOKEN_CONSTANT);
+
+	fCurrentToken.value.SetTo((int64)strtoull(
+			fCurrentToken.string.String(), NULL, 16));
+	return fCurrentToken;
+}
+
+
+int32
+Tokenizer::_CurrentPos() const
+{
+	return fCurrentChar - fString.String();
+}

src/apps/debugger/source_language/expression_evaluators/CLanguageTokenizer.h

@@ -0,0 +1,120 @@
+/*
+ * Copyright 2006-2014 Haiku, Inc. All Rights Reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Stephan Aßmus <[email protected]>
+ *		Rene Gollent <[email protected]>
+ *		John Scipione <[email protected]>
+ *		Ingo Weinhold <[email protected]>
+ */
+#ifndef C_LANGUAGE_TOKENIZER
+#define C_LANGUAGE_TOKENIZER
+
+
+#include <String.h>
+
+#include <Variant.h>
+
+
+namespace CLanguage {
+
+
+enum {
+	TOKEN_NONE					= 0,
+	TOKEN_IDENTIFIER,
+	TOKEN_CONSTANT,
+	TOKEN_END_OF_LINE,
+
+	TOKEN_PLUS,
+	TOKEN_MINUS,
+
+	TOKEN_STAR,
+	TOKEN_SLASH,
+	TOKEN_MODULO,
+
+	TOKEN_POWER,
+
+	TOKEN_OPENING_BRACKET,
+	TOKEN_CLOSING_BRACKET,
+
+	TOKEN_LOGICAL_AND,
+	TOKEN_LOGICAL_OR,
+	TOKEN_LOGICAL_NOT,
+	TOKEN_BITWISE_AND,
+	TOKEN_BITWISE_OR,
+	TOKEN_BITWISE_NOT,
+	TOKEN_BITWISE_XOR,
+	TOKEN_EQ,
+	TOKEN_NE,
+	TOKEN_GT,
+	TOKEN_GE,
+	TOKEN_LT,
+	TOKEN_LE,
+
+	TOKEN_MEMBER_PTR
+};
+
+
+class ParseException {
+ public:
+	ParseException(const char* message, int32 position)
+		: message(message),
+		  position(position)
+	{
+	}
+
+	ParseException(const ParseException& other)
+		: message(other.message),
+		  position(other.position)
+	{
+	}
+
+	BString	message;
+	int32	position;
+};
+
+
+struct Token {
+								Token();
+								Token(const Token& other);
+								Token(const char* string, int32 length,
+								int32 position, int32 type);
+			Token& 	operator=(const Token& other);
+
+	BString						string;
+	int32						type;
+	BVariant					value;
+	int32						position;
+};
+
+
+class Tokenizer {
+public:
+								Tokenizer();
+
+			void 				SetTo(const char* string);
+
+			const Token& 		NextToken();
+			void 				RewindToken();
+private:
+			bool 				_ParseOperator();
+ 			char 				_Peek() const;
+
+	static 	bool 				_IsHexDigit(char c);
+
+			Token& 				_ParseHexOperand();
+			int32 				_CurrentPos() const;
+
+private:
+	BString						fString;
+	const char*					fCurrentChar;
+	Token						fCurrentToken;
+	bool						fReuseToken;
+};
+
+
+}	// namespace CLanguage
+
+
+#endif	// C_LANGUAGE_TOKENIZER