From 08e5249142cc88561e8219e41deeefd3ef5acc80 Mon Sep 17 00:00:00 2001 From: Adrien Destugues Date: Sun, 25 Sep 2016 23:16:19 +0200 Subject: [PATCH] TextEncoding: add Encode/Decode methods. This replaces convert_from/to_utf8 in a way that doesn't leak memory. Use the new method in StyledEdit. Fixes #6252, #3065. --- headers/os/locale/TextEncoding.h | 20 ++- .../translators/stxt/STXTTranslator.cpp | 39 ++--- src/kits/locale/TextEncoding.cpp | 143 ++++++++++++++++++ 3 files changed, 182 insertions(+), 20 deletions(-) diff --git a/headers/os/locale/TextEncoding.h b/headers/os/locale/TextEncoding.h index 635a8e81b1..0453e193bb 100644 --- a/headers/os/locale/TextEncoding.h +++ b/headers/os/locale/TextEncoding.h @@ -13,16 +13,32 @@ #include +struct UConverter; + + class TextEncoding { public: + TextEncoding(BString name); TextEncoding(const char* data, size_t length); - BString GetName(); + ~TextEncoding(); + + status_t InitCheck(); + BString GetName(); + + status_t Encode(const char* input, size_t& inputLength, char* output, + size_t& outputLength); + status_t Decode(const char* input, size_t& inputLength, char* output, + size_t& outputLength); + status_t Flush(char* output, size_t& outputLength); private: BString fName; + + UConverter* fUtf8Converter; + UConverter* fConverter; }; -#endif /* !TEXTENCODING_H */ +#endif /* TEXTENCODING_H */ diff --git a/src/add-ons/translators/stxt/STXTTranslator.cpp b/src/add-ons/translators/stxt/STXTTranslator.cpp index d1da5d8cf2..be4e58c62d 100644 --- a/src/add-ons/translators/stxt/STXTTranslator.cpp +++ b/src/add-ons/translators/stxt/STXTTranslator.cpp @@ -522,40 +522,40 @@ translate_from_text(BPositionIO* source, const char* encoding, bool forceEncodin size_t fSize; } encodingBuffer; BMallocIO encodingIO; - uint32 encodingID = 0; - // defaults to UTF-8 or no encoding BNode* node = dynamic_cast(source); if (node != NULL) { // determine encoding, if available - const BCharacterSet* characterSet = NULL; bool hasAttribute = false; if (encoding != NULL && !forceEncoding) { BString name; - if (node->ReadAttrString("be:encoding", &name) == B_OK) { + if ((node->ReadAttrString("be:encoding", &name) == B_OK) + && (name.Length() > 0)) { encoding = name.String(); hasAttribute = true; } else { + // Try the BeOS version of the atribute, which used an int32 + // and a well-known list of encodings. int32 value; ssize_t bytesRead = node->ReadAttr("be:encoding", B_INT32_TYPE, 0, &value, sizeof(value)); if (bytesRead == (ssize_t)sizeof(value)) { hasAttribute = true; - if (value != 65535) + if (value != 65535) { + const BCharacterSet* characterSet = NULL; characterSet = BCharacterSetRoster::GetCharacterSetByConversionID(value); + if (characterSet != NULL) + encoding = characterSet->GetName(); + } } } } else { hasAttribute = true; // we don't write the encoding in this case } - if (characterSet == NULL && encoding != NULL) - characterSet = BCharacterSetRoster::FindCharacterSetByName(encoding); - if (characterSet != NULL) { - encodingID = characterSet->GetConversionID(); + if (encoding != NULL) encodingBuffer.Allocate(READ_BUFFER_SIZE * 4); - } if (!hasAttribute && encoding != NULL) { // add encoding attribute, so that someone opening the file can @@ -567,7 +567,8 @@ translate_from_text(BPositionIO* source, const char* encoding, bool forceEncodin off_t outputSize = 0; ssize_t bytesRead; - int32 state = 0; + + TextEncoding codec(encoding); // output the actual text part of the data do { @@ -591,22 +592,24 @@ translate_from_text(BPositionIO* source, const char* encoding, bool forceEncodin outputSize += bytesRead; } else { // decode text file to UTF-8 - char* pos = (char*)buffer; - int32 encodingLength = encodingIO.BufferLength(); + const char* pos = (char*)buffer; + size_t encodingLength = encodingIO.BufferLength(); int32 bytesLeft = bytesRead; - int32 bytes; + size_t bytes; do { encodingLength = READ_BUFFER_SIZE * 4; bytes = bytesLeft; - status = convert_to_utf8(encodingID, pos, &bytes, - (char*)encodingBuffer.Buffer(), &encodingLength, &state); - if (status < B_OK) + status = codec.Decode(pos, bytes, + (char*)encodingBuffer.Buffer(), encodingLength); + if (status < B_OK) { + puts("oops"); return status; + } ssize_t bytesWritten = destination->Write(encodingBuffer.Buffer(), encodingLength); - if (bytesWritten < encodingLength) { + if (bytesWritten < (ssize_t)encodingLength) { if (bytesWritten < B_OK) return bytesWritten; diff --git a/src/kits/locale/TextEncoding.cpp b/src/kits/locale/TextEncoding.cpp index 1cc25b39b7..5700a50408 100644 --- a/src/kits/locale/TextEncoding.cpp +++ b/src/kits/locale/TextEncoding.cpp @@ -6,10 +6,25 @@ #include "TextEncoding.h" +#include #include +#include + + +TextEncoding::TextEncoding(BString name) + : + fName(name), + fUtf8Converter(NULL), + fConverter(NULL) +{ +} + TextEncoding::TextEncoding(const char* data, size_t length) + : + fUtf8Converter(NULL), + fConverter(NULL) { UErrorCode error = U_ZERO_ERROR; @@ -22,6 +37,134 @@ TextEncoding::TextEncoding(const char* data, size_t length) } +TextEncoding::~TextEncoding() +{ + if (fUtf8Converter != NULL) + ucnv_close(fUtf8Converter); + + if (fConverter != NULL) + ucnv_close(fConverter); +} + + +status_t +TextEncoding::InitCheck() +{ + if (fName.IsEmpty()) + return B_NO_INIT; + else + return B_OK; +} + + +status_t +TextEncoding::Decode(const char* input, size_t& inputLength, char* output, + size_t& outputLength) +{ + const char* base = input; + char* target = output; + + // Optimize the easy case. + // Note: we don't check the input to be valid UTF-8 when doing that. + if (fName == "UTF-8") { + outputLength = std::min(inputLength, outputLength); + inputLength = outputLength; + memcpy(output, input, inputLength); + return B_OK; + } + + UErrorCode error = U_ZERO_ERROR; + + if (fUtf8Converter == NULL) + fUtf8Converter = ucnv_open("UTF-8", &error); + + if (fConverter == NULL) + fConverter = ucnv_open(fName.String(), &error); + + ucnv_convertEx(fUtf8Converter, fConverter, &target, output + outputLength, + &base, input + inputLength, NULL, NULL, NULL, NULL, FALSE, TRUE, + &error); + if (!U_SUCCESS(error)) { + printf("zz %s\n", u_errorName(error)); + } + + + // inputLength is set to the number of bytes consumed. We may not use all of + // the input data (for example if it is cut in the middle of an utf-8 char). + inputLength = base - input; + outputLength = target - output; + + if (!U_SUCCESS(error)) + return B_ERROR; + + return B_OK; +} + + +status_t +TextEncoding::Encode(const char* input, size_t& inputLength, char* output, + size_t& outputLength) +{ + const char* base = input; + char* target = output; + + // Optimize the easy case. + // Note: we don't check the input to be valid UTF-8 when doing that. + if (fName == "UTF-8") { + outputLength = std::min(inputLength, outputLength); + inputLength = outputLength; + memcpy(output, input, inputLength); + return B_OK; + } + + UErrorCode error = U_ZERO_ERROR; + + if (fUtf8Converter == NULL) + fUtf8Converter = ucnv_open("UTF-8", &error); + + if (fConverter == NULL) + fConverter = ucnv_open(fName.String(), &error); + + ucnv_convertEx(fConverter, fUtf8Converter, &target, output + outputLength, + &base, input + inputLength, NULL, NULL, NULL, NULL, FALSE, TRUE, + &error); + + // inputLength is set to the number of bytes consumed. We may not use all of + // the input data (for example if it is cut in the middle of an utf-8 char). + inputLength = base - input; + outputLength = target - output; + + if (!U_SUCCESS(error)) + return B_ERROR; + + return B_OK; +} + + +status_t +TextEncoding::Flush(char* output, size_t& outputLength) +{ + char* target = output; + + if (fName == "UTF-8") + return B_OK; + + if (fUtf8Converter == NULL || fConverter == NULL) + return B_NO_INIT; + + UErrorCode error = U_ZERO_ERROR; + + ucnv_convertEx(fConverter, fUtf8Converter, &target, output + outputLength, + NULL, NULL, NULL, NULL, NULL, NULL, FALSE, TRUE, + &error); + + if (!U_SUCCESS(error)) + return B_ERROR; + + return B_OK; +} + + BString TextEncoding::GetName() {