Debugger: Fix tokenizer bug.

- When detecting a possible start of string literal, ensure that a
  terminator was actually found, otherwise simply consider it as a
  single character token. Otherwise, cases such as an apostrophe in
  a C-style comment that spanned a single line would result in us
  missing the token for the comment close, and highlighting all
  subsequent lines until such a token was encountered as comments.

src/apps/debugger/source_language/c_family/CLanguageTokenizer.cpp

@@ -172,17 +172,27 @@ Tokenizer::NextToken()
 		fCurrentToken = Token(begin, length, _CurrentPos() - length,
 			TOKEN_IDENTIFIER);
 	} else if (*fCurrentChar == '"' || *fCurrentChar == '\'') {
+		bool terminatorFound = false;
 		const char* begin = fCurrentChar++;
 		while (*fCurrentChar != 0) {
 			if (*fCurrentChar == '\\') {
 				if (*(fCurrentChar++) != 0)
 					fCurrentChar++;
-			} else if (*(fCurrentChar++) == *begin)
+			} else if (*(fCurrentChar++) == *begin) {
+				terminatorFound = true;
 				break;
+			}
 		}
+		int32 tokenType = TOKEN_STRING_LITERAL;
+		if (!terminatorFound) {
+			tokenType = *begin == '"' ? TOKEN_DOUBLE_QUOTE
+					: TOKEN_SINGLE_QUOTE;
+			fCurrentChar = begin + 1;
+		}
+
 		int32 length = fCurrentChar - begin;
 		fCurrentToken = Token(begin, length, _CurrentPos() - length,
-			TOKEN_STRING_LITERAL);
+			tokenType);
 	} else {
 		if (!_ParseOperator()) {
 			int32 type = TOKEN_NONE;

src/apps/debugger/source_language/c_family/CLanguageTokenizer.h

@@ -65,6 +65,9 @@ enum {
 	TOKEN_PERIOD,
 	TOKEN_POUND,
 
+	TOKEN_SINGLE_QUOTE,
+	TOKEN_DOUBLE_QUOTE,
+
 	TOKEN_STRING_LITERAL,
 	TOKEN_BEGIN_COMMENT_BLOCK,
 	TOKEN_END_COMMENT_BLOCK,