From 15852b070a9c6758136665a0dcae2a60854349d1 Mon Sep 17 00:00:00 2001
From: Rene Gollent <rene@gollent.com>
Date: Sun, 28 Dec 2014 23:31:25 -0500
Subject: [PATCH] Debugger: Fix tokenizer bug.

- When detecting a possible start of string literal, ensure that a
  terminator was actually found, otherwise simply consider it as a
  single character token. Otherwise, cases such as an apostrophe in
  a C-style comment that spanned a single line would result in us
  missing the token for the comment close, and highlighting all
  subsequent lines until such a token was encountered as comments.
---
 .../c_family/CLanguageTokenizer.cpp                | 14 ++++++++++++--
 .../source_language/c_family/CLanguageTokenizer.h  |  3 +++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/apps/debugger/source_language/c_family/CLanguageTokenizer.cpp b/src/apps/debugger/source_language/c_family/CLanguageTokenizer.cpp
index 07de4122e3..18eb1415de 100644
--- a/src/apps/debugger/source_language/c_family/CLanguageTokenizer.cpp
+++ b/src/apps/debugger/source_language/c_family/CLanguageTokenizer.cpp
@@ -172,17 +172,27 @@ Tokenizer::NextToken()
 		fCurrentToken = Token(begin, length, _CurrentPos() - length,
 			TOKEN_IDENTIFIER);
 	} else if (*fCurrentChar == '"' || *fCurrentChar == '\'') {
+		bool terminatorFound = false;
 		const char* begin = fCurrentChar++;
 		while (*fCurrentChar != 0) {
 			if (*fCurrentChar == '\\') {
 				if (*(fCurrentChar++) != 0)
 					fCurrentChar++;
-			} else if (*(fCurrentChar++) == *begin)
+			} else if (*(fCurrentChar++) == *begin) {
+				terminatorFound = true;
 				break;
+			}
 		}
+		int32 tokenType = TOKEN_STRING_LITERAL;
+		if (!terminatorFound) {
+			tokenType = *begin == '"' ? TOKEN_DOUBLE_QUOTE
+					: TOKEN_SINGLE_QUOTE;
+			fCurrentChar = begin + 1;
+		}
+
 		int32 length = fCurrentChar - begin;
 		fCurrentToken = Token(begin, length, _CurrentPos() - length,
-			TOKEN_STRING_LITERAL);
+			tokenType);
 	} else {
 		if (!_ParseOperator()) {
 			int32 type = TOKEN_NONE;
diff --git a/src/apps/debugger/source_language/c_family/CLanguageTokenizer.h b/src/apps/debugger/source_language/c_family/CLanguageTokenizer.h
index 0fcb3193c9..ecd900baaa 100644
--- a/src/apps/debugger/source_language/c_family/CLanguageTokenizer.h
+++ b/src/apps/debugger/source_language/c_family/CLanguageTokenizer.h
@@ -65,6 +65,9 @@ enum {
 	TOKEN_PERIOD,
 	TOKEN_POUND,
 
+	TOKEN_SINGLE_QUOTE,
+	TOKEN_DOUBLE_QUOTE,
+
 	TOKEN_STRING_LITERAL,
 	TOKEN_BEGIN_COMMENT_BLOCK,
 	TOKEN_END_COMMENT_BLOCK,