TextEncoding: add Encode/Decode methods.

This replaces convert_from/to_utf8 in a way that doesn't leak memory.

Use the new method in StyledEdit.

Fixes #6252, #3065.

headers/os/locale/TextEncoding.h

@@ -13,16 +13,32 @@
 #include <stddef.h>
 
 
+struct UConverter;
+
+
 class TextEncoding
 {
 	public:
+		TextEncoding(BString name);
 		TextEncoding(const char* data, size_t length);
 
-		BString GetName();
+		~TextEncoding();
+
+		status_t	InitCheck();
+		BString		GetName();
+
+		status_t Encode(const char* input, size_t& inputLength, char* output,
+			size_t& outputLength);
+		status_t Decode(const char* input, size_t& inputLength, char* output,
+			size_t& outputLength);
+		status_t Flush(char* output, size_t& outputLength);
 
 	private:
 		BString fName;
+
+		UConverter* fUtf8Converter;
+		UConverter* fConverter;
 };
 
 
-#endif /* !TEXTENCODING_H */
+#endif /* TEXTENCODING_H */

src/add-ons/translators/stxt/STXTTranslator.cpp

@@ -522,40 +522,40 @@ translate_from_text(BPositionIO* source, const char* encoding, bool forceEncodin
 			size_t	fSize;
 	} encodingBuffer;
 	BMallocIO encodingIO;
-	uint32 encodingID = 0;
-		// defaults to UTF-8 or no encoding
 
 	BNode* node = dynamic_cast<BNode*>(source);
 	if (node != NULL) {
 		// determine encoding, if available
-		const BCharacterSet* characterSet = NULL;
 		bool hasAttribute = false;
 		if (encoding != NULL && !forceEncoding) {
 			BString name;
-			if (node->ReadAttrString("be:encoding", &name) == B_OK) {
+			if ((node->ReadAttrString("be:encoding", &name) == B_OK)
+				&& (name.Length() > 0)) {
 				encoding = name.String();
 				hasAttribute = true;
 			} else {
+				// Try the BeOS version of the atribute, which used an int32
+				// and a well-known list of encodings.
 				int32 value;
 				ssize_t bytesRead = node->ReadAttr("be:encoding", B_INT32_TYPE, 0,
 					&value, sizeof(value));
 				if (bytesRead == (ssize_t)sizeof(value)) {
 					hasAttribute = true;
-					if (value != 65535)
+					if (value != 65535) {
+						const BCharacterSet* characterSet = NULL;
 						characterSet = BCharacterSetRoster::GetCharacterSetByConversionID(value);
+						if (characterSet != NULL)
+							encoding = characterSet->GetName();
+					}
 				}
 			}
 		} else {
 			hasAttribute = true;
 				// we don't write the encoding in this case
 		}
-		if (characterSet == NULL && encoding != NULL)
-			characterSet = BCharacterSetRoster::FindCharacterSetByName(encoding);
 
-		if (characterSet != NULL) {
+		if (encoding != NULL)
-			encodingID = characterSet->GetConversionID();
 			encodingBuffer.Allocate(READ_BUFFER_SIZE * 4);
-		}
 
 		if (!hasAttribute && encoding != NULL) {
 			// add encoding attribute, so that someone opening the file can
@@ -567,7 +567,8 @@ translate_from_text(BPositionIO* source, const char* encoding, bool forceEncodin
 
 	off_t outputSize = 0;
 	ssize_t bytesRead;
-	int32 state = 0;
+
+	TextEncoding codec(encoding);
 
 	// output the actual text part of the data
 	do {
@@ -591,22 +592,24 @@ translate_from_text(BPositionIO* source, const char* encoding, bool forceEncodin
 			outputSize += bytesRead;
 		} else {
 			// decode text file to UTF-8
-			char* pos = (char*)buffer;
+			const char* pos = (char*)buffer;
-			int32 encodingLength = encodingIO.BufferLength();
+			size_t encodingLength = encodingIO.BufferLength();
 			int32 bytesLeft = bytesRead;
-			int32 bytes;
+			size_t bytes;
 			do {
 				encodingLength = READ_BUFFER_SIZE * 4;
 				bytes = bytesLeft;
 
-				status = convert_to_utf8(encodingID, pos, &bytes,
+				status = codec.Decode(pos, bytes,
-					(char*)encodingBuffer.Buffer(), &encodingLength, &state);
+					(char*)encodingBuffer.Buffer(), encodingLength);
-				if (status < B_OK)
+				if (status < B_OK) {
+					puts("oops");
 					return status;
+				}
 
 				ssize_t bytesWritten = destination->Write(encodingBuffer.Buffer(),
 					encodingLength);
-				if (bytesWritten < encodingLength) {
+				if (bytesWritten < (ssize_t)encodingLength) {
 					if (bytesWritten < B_OK)
 						return bytesWritten;
 

src/kits/locale/TextEncoding.cpp

@@ -6,10 +6,25 @@
 
 #include "TextEncoding.h"
 
+#include <unicode/ucnv.h>
 #include <unicode/ucsdet.h>
 
+#include <algorithm>
+
+
+TextEncoding::TextEncoding(BString name)
+	:
+	fName(name),
+	fUtf8Converter(NULL),
+	fConverter(NULL)
+{
+}
+
 
 TextEncoding::TextEncoding(const char* data, size_t length)
+	:
+	fUtf8Converter(NULL),
+	fConverter(NULL)
 {
 	UErrorCode error = U_ZERO_ERROR;
 
@@ -22,6 +37,134 @@ TextEncoding::TextEncoding(const char* data, size_t length)
 }
 
 
+TextEncoding::~TextEncoding()
+{
+	if (fUtf8Converter != NULL)
+		ucnv_close(fUtf8Converter);
+
+	if (fConverter != NULL)
+		ucnv_close(fConverter);
+}
+
+
+status_t
+TextEncoding::InitCheck()
+{
+	if (fName.IsEmpty())
+		return B_NO_INIT;
+	else
+		return B_OK;
+}
+
+
+status_t
+TextEncoding::Decode(const char* input, size_t& inputLength, char* output,
+	size_t& outputLength)
+{
+	const char* base = input;
+	char* target = output;
+
+	// Optimize the easy case.
+	// Note: we don't check the input to be valid UTF-8 when doing that.
+	if (fName == "UTF-8") {
+		outputLength = std::min(inputLength, outputLength);
+		inputLength = outputLength;
+		memcpy(output, input, inputLength);
+		return B_OK;
+	}
+
+	UErrorCode error = U_ZERO_ERROR;
+
+	if (fUtf8Converter == NULL)
+		fUtf8Converter = ucnv_open("UTF-8", &error);
+
+	if (fConverter == NULL)
+		fConverter = ucnv_open(fName.String(), &error);
+
+	ucnv_convertEx(fUtf8Converter, fConverter, &target, output + outputLength,
+		&base, input + inputLength, NULL, NULL, NULL, NULL, FALSE, TRUE,
+		&error);
+	if (!U_SUCCESS(error)) {
+		printf("zz %s\n", u_errorName(error));
+	}
+
+
+	// inputLength is set to the number of bytes consumed. We may not use all of
+	// the input data (for example if it is cut in the middle of an utf-8 char).
+	inputLength = base - input;
+	outputLength = target - output;
+
+	if (!U_SUCCESS(error))
+		return B_ERROR;
+
+	return B_OK;
+}
+
+
+status_t
+TextEncoding::Encode(const char* input, size_t& inputLength, char* output,
+	size_t& outputLength)
+{
+	const char* base = input;
+	char* target = output;
+
+	// Optimize the easy case.
+	// Note: we don't check the input to be valid UTF-8 when doing that.
+	if (fName == "UTF-8") {
+		outputLength = std::min(inputLength, outputLength);
+		inputLength = outputLength;
+		memcpy(output, input, inputLength);
+		return B_OK;
+	}
+
+	UErrorCode error = U_ZERO_ERROR;
+
+	if (fUtf8Converter == NULL)
+		fUtf8Converter = ucnv_open("UTF-8", &error);
+
+	if (fConverter == NULL)
+		fConverter = ucnv_open(fName.String(), &error);
+
+	ucnv_convertEx(fConverter, fUtf8Converter, &target, output + outputLength,
+		&base, input + inputLength, NULL, NULL, NULL, NULL, FALSE, TRUE,
+		&error);
+
+	// inputLength is set to the number of bytes consumed. We may not use all of
+	// the input data (for example if it is cut in the middle of an utf-8 char).
+	inputLength = base - input;
+	outputLength = target - output;
+
+	if (!U_SUCCESS(error))
+		return B_ERROR;
+
+	return B_OK;
+}
+
+
+status_t
+TextEncoding::Flush(char* output, size_t& outputLength)
+{
+	char* target = output;
+
+	if (fName == "UTF-8")
+		return B_OK;
+
+	if (fUtf8Converter == NULL || fConverter == NULL)
+		return B_NO_INIT;
+
+	UErrorCode error = U_ZERO_ERROR;
+
+	ucnv_convertEx(fConverter, fUtf8Converter, &target, output + outputLength,
+		NULL, NULL, NULL, NULL, NULL, NULL, FALSE, TRUE,
+		&error);
+
+	if (!U_SUCCESS(error))
+		return B_ERROR;
+
+	return B_OK;
+}
+
+
 BString
 TextEncoding::GetName()
 {