diff --git a/build/jam/OptionalPackages b/build/jam/OptionalPackages index 98dced5a9d..f120cfb2a9 100644 --- a/build/jam/OptionalPackages +++ b/build/jam/OptionalPackages @@ -536,8 +536,8 @@ if [ IsOptionalHaikuImagePackageAdded DevelopmentBase ] if $(HAIKU_GCC_VERSION[1]) = 4 || $(isHybridBuild) { InstallOptionalHaikuImagePackage - gcc-4.5.3-x86-gcc4-2011-11-22.zip - : $(baseURL)/gcc-4.5.3-x86-gcc4-2011-11-22.zip ; + gcc-4.5.3-x86-gcc4-2011-12-14.zip + : $(baseURL)/gcc-4.5.3-x86-gcc4-2011-12-14.zip ; } if $(HAIKU_GCC_VERSION[1]) = 4 { @@ -549,7 +549,7 @@ if [ IsOptionalHaikuImagePackageAdded DevelopmentBase ] local libs = libstdc++.so libsupc++.so ; for lib in $(libs) { AddSymlinkToHaikuHybridImage - develop abi x86 gcc4 tools gcc-4.5.3-haiku-111122 lib + develop abi x86 gcc4 tools gcc-4.5.3-haiku-111214 lib : /system/lib $(lib) : : true ; } } diff --git a/headers/posix/wchar.h b/headers/posix/wchar.h index d7c604ba37..16bf7b81f8 100644 --- a/headers/posix/wchar.h +++ b/headers/posix/wchar.h @@ -27,8 +27,10 @@ typedef __WINT_TYPE__ wint_t; typedef int wctype_t; typedef struct { + void* converter; + char charset[64]; unsigned int count; - unsigned int converterID; + char data[1024 + 8]; // 1024 bytes for data, 8 for alignment space } mbstate_t; diff --git a/headers/private/libroot/locale/ICUCategoryData.h b/headers/private/libroot/locale/ICUCategoryData.h index ad8088b804..dd25d57a3b 100644 --- a/headers/private/libroot/locale/ICUCategoryData.h +++ b/headers/private/libroot/locale/ICUCategoryData.h @@ -8,12 +8,11 @@ #include #include -#include #include #include -#include "ICUConverterManager.h" +#include "ICUThreadLocalStorageValue.h" namespace BPrivate { @@ -38,7 +37,7 @@ protected: char* destination, int destinationSize, const char* defaultValue = ""); - status_t _GetConverter(ICUConverterRef& converterRefOut); + status_t _GetConverter(UConverter*& converterOut); static const uint16 skMaxPosixLocaleNameLen = 128; static const size_t skLCBufSize = 16; diff --git a/headers/private/libroot/locale/ICUConverterManager.h b/headers/private/libroot/locale/ICUConverterManager.h deleted file mode 100644 index 68d0d5a257..0000000000 --- a/headers/private/libroot/locale/ICUConverterManager.h +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright 2011, Oliver Tappe, zooey@hirschkaefer.de. - * Distributed under the terms of the MIT License. - */ -#ifndef _ICU_CONVERTER_MANAGER_H -#define _ICU_CONVERTER_MANAGER_H - - -#include - -#include - -#include - -#include - -#include -#include -#include -//#include - -#include "ICUThreadLocalStorageValue.h" - - -namespace BPrivate { -namespace Libroot { - - -class ICUConverterInfo : public BReferenceable { -public: - ICUConverterInfo(UConverter* converter, - const char* charset, ICUConverterID id); - virtual ~ICUConverterInfo(); - - UConverter* Converter() const - { return fConverter; } - - const char* Charset() const - { return fCharset; } - - ICUConverterID ID() const - { return fID; } - -private: - UConverter* fConverter; - char fCharset[UCNV_MAX_CONVERTER_NAME_LENGTH]; - ICUConverterID fID; -}; - - -typedef BReference ICUConverterRef; - - -class ICUConverterManager { -public: - ICUConverterManager(); - ~ICUConverterManager(); - - status_t CreateConverter(const char* charset, - ICUConverterRef& converterRefOut, - ICUConverterID& idOut); - - status_t GetConverter(ICUConverterID id, - ICUConverterRef& converterRefOut); - - status_t DropConverter(ICUConverterID id); - - static ICUConverterManager* Instance(); - -private: - static void _CreateInstance(); - - static ICUConverterManager* sInstance; - - static const size_t skMaxConvertersPerProcess = 1024; - -private: - class LinkedConverterInfo - : - public ICUConverterInfo, - public DoublyLinkedListLinkImpl - { - public: - LinkedConverterInfo(UConverter* converter, const char* charset, - ICUConverterID id) - : - ICUConverterInfo(converter, charset, id) - { - } - }; - typedef std::map ConverterMap; - typedef DoublyLinkedList ConverterList; - -private: - ConverterMap fConverterMap; - ConverterList fLRUConverters; - mutex fMutex; - ICUConverterID fNextConverterID; -}; - - -} // namespace Libroot -} // namespace BPrivate - - -#endif // _ICU_CONVERTER_MANAGER_H diff --git a/headers/private/libroot/locale/ICUCtypeData.h b/headers/private/libroot/locale/ICUCtypeData.h index ebf60169ff..72b335ee9a 100644 --- a/headers/private/libroot/locale/ICUCtypeData.h +++ b/headers/private/libroot/locale/ICUCtypeData.h @@ -49,7 +49,7 @@ public: private: status_t _GetConverterForMbState(mbstate_t* mbState, - ICUConverterRef& converterRefOut); + UConverter*& converterOut); status_t _DropConverterFromMbState(mbstate_t* mbState); diff --git a/headers/private/libroot/locale/ICUThreadLocalStorageValue.h b/headers/private/libroot/locale/ICUThreadLocalStorageValue.h index 5ea3c26264..3006a45a10 100644 --- a/headers/private/libroot/locale/ICUThreadLocalStorageValue.h +++ b/headers/private/libroot/locale/ICUThreadLocalStorageValue.h @@ -8,6 +8,8 @@ #include +#include + #include @@ -15,11 +17,9 @@ namespace BPrivate { namespace Libroot { -typedef unsigned int ICUConverterID; - - struct ICUThreadLocalStorageValue { - ICUConverterID converterID; + UConverter* converter; + char charset[64]; ICUThreadLocalStorageValue(); ~ICUThreadLocalStorageValue(); diff --git a/src/system/libroot/add-ons/icu/ICUCategoryData.cpp b/src/system/libroot/add-ons/icu/ICUCategoryData.cpp index 88e3dd1062..1a7af42775 100644 --- a/src/system/libroot/add-ons/icu/ICUCategoryData.cpp +++ b/src/system/libroot/add-ons/icu/ICUCategoryData.cpp @@ -29,44 +29,6 @@ ICUCategoryData::~ICUCategoryData() } -status_t -ICUCategoryData::_GetConverter(ICUConverterRef& converterRefOut) -{ - // we use different converter-IDs per thread in order to avoid converters - // being used by more than one thread - ICUThreadLocalStorageValue* tlsValue = NULL; - status_t result = ICUThreadLocalStorageValue::GetInstanceForKey( - fThreadLocalStorageKey, tlsValue); - if (result != B_OK) - return result; - - ICUConverterRef converterRef; - result = ICUConverterManager::Instance()->GetConverter( - tlsValue->converterID, converterRef); - if (result == B_OK) { - if (strcmp(converterRef->Charset(), fGivenCharset) == 0) { - converterRefOut = converterRef; - return B_OK; - } - - // charset no longer matches the converter, we need to dump it and - // create a new one - ICUConverterManager::Instance()->DropConverter(tlsValue->converterID); - tlsValue->converterID = 0; - } - - // create a new converter for the current charset - result = ICUConverterManager::Instance()->CreateConverter(fGivenCharset, - converterRef, tlsValue->converterID); - if (result != B_OK) - return result; - - converterRefOut = converterRef; - - return B_OK; -} - - status_t ICUCategoryData::SetTo(const Locale& locale, const char* posixLocaleName) { @@ -115,14 +77,14 @@ ICUCategoryData::_ConvertUnicodeStringToLocaleconvEntry( const UnicodeString& string, char* destination, int destinationSize, const char* defaultValue) { - ICUConverterRef converterRef; - status_t result = _GetConverter(converterRef); + UConverter* converter; + status_t result = _GetConverter(converter); if (result != B_OK) return result; UErrorCode icuStatus = U_ZERO_ERROR; - ucnv_fromUChars(converterRef->Converter(), destination, destinationSize, - string.getBuffer(), string.length(), &icuStatus); + ucnv_fromUChars(converter, destination, destinationSize, string.getBuffer(), + string.length(), &icuStatus); if (!U_SUCCESS(icuStatus)) { switch (icuStatus) { case U_BUFFER_OVERFLOW_ERROR: @@ -144,11 +106,63 @@ ICUCategoryData::_ConvertUnicodeStringToLocaleconvEntry( } +status_t +ICUCategoryData::_GetConverter(UConverter*& converterOut) +{ + // we use different converters per thread to avoid concurrent accesses + ICUThreadLocalStorageValue* tlsValue = NULL; + status_t result = ICUThreadLocalStorageValue::GetInstanceForKey( + fThreadLocalStorageKey, tlsValue); + if (result != B_OK) + return result; + + if (tlsValue->converter != NULL) { + if (strcmp(tlsValue->charset, fGivenCharset) == 0) { + converterOut = tlsValue->converter; + return B_OK; + } + + // charset no longer matches the converter, we need to dump it and + // create a new one + ucnv_close(tlsValue->converter); + } + + // create a new converter for the current charset + UErrorCode icuStatus = U_ZERO_ERROR; + UConverter* icuConverter = ucnv_open(fGivenCharset, &icuStatus); + if (icuConverter == NULL) + return B_NAME_NOT_FOUND; + + // setup the new converter to stop upon any errors + icuStatus = U_ZERO_ERROR; + ucnv_setToUCallBack(icuConverter, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, + &icuStatus); + if (!U_SUCCESS(icuStatus)) { + ucnv_close(icuConverter); + return B_ERROR; + } + icuStatus = U_ZERO_ERROR; + ucnv_setFromUCallBack(icuConverter, UCNV_FROM_U_CALLBACK_STOP, NULL, NULL, + NULL, &icuStatus); + if (!U_SUCCESS(icuStatus)) { + ucnv_close(icuConverter); + return B_ERROR; + } + + tlsValue->converter = icuConverter; + strlcpy(tlsValue->charset, fGivenCharset, sizeof(tlsValue->charset)); + + converterOut = icuConverter; + + return B_OK; +} + + status_t ICUCategoryData::_SetupConverter() { - ICUConverterRef converterRef; - return _GetConverter(converterRef); + UConverter* converter; + return _GetConverter(converter); } diff --git a/src/system/libroot/add-ons/icu/ICUCollateData.cpp b/src/system/libroot/add-ons/icu/ICUCollateData.cpp index a9d9264f22..70e88e4026 100644 --- a/src/system/libroot/add-ons/icu/ICUCollateData.cpp +++ b/src/system/libroot/add-ons/icu/ICUCollateData.cpp @@ -10,8 +10,6 @@ #include -#include "ICUConverterManager.h" - namespace BPrivate { namespace Libroot { @@ -153,14 +151,13 @@ ICUCollateData::_ToUnicodeString(const char* in, UnicodeString& out) if (inLen == 0) return B_OK; - ICUConverterRef converterRef; - status_t result = _GetConverter(converterRef); + UConverter* converter; + status_t result = _GetConverter(converter); if (result != B_OK) return result; UErrorCode icuStatus = U_ZERO_ERROR; - int32_t outLen = ucnv_toUChars(converterRef->Converter(), NULL, 0, in, - inLen, &icuStatus); + int32_t outLen = ucnv_toUChars(converter, NULL, 0, in, inLen, &icuStatus); if (icuStatus != U_BUFFER_OVERFLOW_ERROR) return B_BAD_VALUE; if (outLen < 0) @@ -170,8 +167,8 @@ ICUCollateData::_ToUnicodeString(const char* in, UnicodeString& out) UChar* outBuf = out.getBuffer(outLen + 1); icuStatus = U_ZERO_ERROR; - outLen = ucnv_toUChars(converterRef->Converter(), outBuf, outLen + 1, in, - inLen, &icuStatus); + outLen + = ucnv_toUChars(converter, outBuf, outLen + 1, in, inLen, &icuStatus); if (!U_SUCCESS(icuStatus)) { out.releaseBuffer(0); return B_BAD_VALUE; diff --git a/src/system/libroot/add-ons/icu/ICUCtypeData.cpp b/src/system/libroot/add-ons/icu/ICUCtypeData.cpp index b47fd0f0e7..9fd3ff22b9 100644 --- a/src/system/libroot/add-ons/icu/ICUCtypeData.cpp +++ b/src/system/libroot/add-ons/icu/ICUCtypeData.cpp @@ -10,10 +10,15 @@ #include #include +#include + #include +#include + //#define TRACE_CTYPE +#undef TRACE #ifdef TRACE_CTYPE # include # define TRACE(x) debug_printf x @@ -58,12 +63,11 @@ ICUCtypeData::SetTo(const Locale& locale, const char* posixLocaleName) UErrorCode icuStatus = U_ZERO_ERROR; - ICUConverterRef converterRef; - result = _GetConverter(converterRef); + UConverter* converter; + result = _GetConverter(converter); if (result != B_OK) return result; - UConverter* converter = converterRef->Converter(); ucnv_reset(converter); fDataBridge->setMbCurMax(ucnv_getMaxCharSize(converter)); @@ -207,16 +211,14 @@ status_t ICUCtypeData::MultibyteToWchar(wchar_t* wcOut, const char* mb, size_t mbLen, mbstate_t* mbState, size_t& lengthOut) { - ICUConverterRef converterRef; - status_t result = _GetConverterForMbState(mbState, converterRef); + UConverter* converter = NULL; + status_t result = _GetConverterForMbState(mbState, converter); if (result != B_OK) { - TRACE(("MultibyteToWchar(): couldn't get converter for ID %d - %lx\n", - mbState->converterID, result)); + TRACE(("MultibyteToWchar(): couldn't get converter for mbstate %p - " + "%lx\n", mbState, result)); return result; } - UConverter* converter = converterRef->Converter(); - // do the conversion UErrorCode icuStatus = U_ZERO_ERROR; @@ -233,8 +235,6 @@ ICUCtypeData::MultibyteToWchar(wchar_t* wcOut, const char* mb, size_t mbLen, icuStatus = U_ZERO_ERROR; } - UChar32 unicodeChar = 0xBADBEEF; - if (!U_SUCCESS(icuStatus)) { // conversion failed because of illegal character sequence TRACE(("MultibyteToWchar(): illegal character sequence\n")); @@ -247,6 +247,7 @@ ICUCtypeData::MultibyteToWchar(wchar_t* wcOut, const char* mb, size_t mbLen, mbState->count = sourceLengthUsed; result = B_BAD_INDEX; } else { + UChar32 unicodeChar = 0xBADBEEF; U16_GET(targetBuffer, 0, 0, 2, unicodeChar); if (unicodeChar == 0) { @@ -274,16 +275,14 @@ ICUCtypeData::MultibyteStringToWchar(wchar_t* wcDest, size_t wcDestLength, const char** mbSource, size_t mbSourceLength, mbstate_t* mbState, size_t& lengthOut) { - ICUConverterRef converterRef; - status_t result = _GetConverterForMbState(mbState, converterRef); + UConverter* converter = NULL; + status_t result = _GetConverterForMbState(mbState, converter); if (result != B_OK) { - TRACE(("MultibyteStringToWchar(): couldn't get converter for ID %d -" - " %lx\n", mbState->converterID, result)); + TRACE(("MultibyteStringToWchar(): couldn't get converter for mbstate %p" + " - %lx\n", mbState, result)); return result; } - UConverter* converter = converterRef->Converter(); - bool wcsIsTerminated = false; const char* source = *mbSource; const char* sourceEnd = source + mbSourceLength; @@ -346,16 +345,14 @@ status_t ICUCtypeData::WcharToMultibyte(char* mbOut, wchar_t wc, mbstate_t* mbState, size_t& lengthOut) { - ICUConverterRef converterRef; - status_t result = _GetConverterForMbState(mbState, converterRef); + UConverter* converter = NULL; + status_t result = _GetConverterForMbState(mbState, converter); if (result != B_OK) { - TRACE(("WcharToMultibyte(): couldn't get converter for ID %d - %lx\n", - mbState->converterID, result)); + TRACE(("WcharToMultibyte(): couldn't get converter for mbstate %p - " + "%lx\n", mbState, result)); return result; } - UConverter* converter = converterRef->Converter(); - // convert input from UTF-32 to UTF-16 UChar ucharBuffer[2]; size_t ucharLength; @@ -409,16 +406,14 @@ ICUCtypeData::WcharStringToMultibyte(char* mbDest, size_t mbDestLength, const wchar_t** wcSource, size_t wcSourceLength, mbstate_t* mbState, size_t& lengthOut) { - ICUConverterRef converterRef; - status_t result = _GetConverterForMbState(mbState, converterRef); + UConverter* converter = NULL; + status_t result = _GetConverterForMbState(mbState, converter); if (result != B_OK) { - TRACE(("WcharStringToMultibyte(): couldn't get converter for ID %d " - "- %lx\n", mbState->converterID, result)); + TRACE(("WcharStringToMultibyte(): couldn't get converter for mbstate %p" + " - %lx\n", mbState, result)); return result; } - UConverter* converter = converterRef->Converter(); - bool mbsIsTerminated = false; const UChar32* source = (UChar32*)*wcSource; @@ -509,29 +504,45 @@ ICUCtypeData::GetLanginfo(int index) status_t ICUCtypeData::_GetConverterForMbState(mbstate_t* mbState, - ICUConverterRef& converterRefOut) + UConverter*& converterOut) { - ICUConverterRef converterRef; - status_t result = ICUConverterManager::Instance()->GetConverter( - mbState->converterID, converterRef); - if (result == B_OK) { - if (strcmp(converterRef->Charset(), fGivenCharset) == 0) { - converterRefOut = converterRef; - return B_OK; - } - - // charset no longer matches the converter, we need to dump it and - // create a new one - _DropConverterFromMbState(mbState); + if (strcmp(mbState->charset, fGivenCharset) == 0 + && (char*)mbState->converter >= mbState->data + && (char*)mbState->converter < mbState->data + 8) { + // charset matches and converter actually lives in *this* mbState, + // so we can use it (if the converter points to the outside, it means + // that the mbstate_t has been copied) + converterOut = (UConverter*)mbState->converter; + return B_OK; } - // create a new converter for the current charset - result = ICUConverterManager::Instance()->CreateConverter(fGivenCharset, - converterRef, mbState->converterID); + // charset no longer matches the converter, we need to dump it and + // create a new one + _DropConverterFromMbState(mbState); + + // create a new converter for the current charset ... + UConverter* icuConverter; + status_t result = _GetConverter(icuConverter); if (result != B_OK) return result; - converterRefOut = converterRef; + // ... and clone it into the mbstate + UErrorCode icuStatus = U_ZERO_ERROR; + int32_t bufferSize = sizeof(mbState->data); + UConverter* clone + = ucnv_safeClone(icuConverter, mbState->data, &bufferSize, &icuStatus); + if (clone == NULL || !U_SUCCESS(icuStatus)) + return B_ERROR; + + if ((char*)clone < mbState->data || (char*)clone >= mbState->data + 8) { + // buffer is too small (shouldn't happen according to ICU docs) + return B_NO_MEMORY; + } + + strlcpy(mbState->charset, fGivenCharset, sizeof(mbState->charset)); + mbState->converter = clone; + + converterOut = clone; return B_OK; } @@ -540,8 +551,9 @@ ICUCtypeData::_GetConverterForMbState(mbstate_t* mbState, status_t ICUCtypeData::_DropConverterFromMbState(mbstate_t* mbState) { - ICUConverterManager::Instance()->DropConverter(mbState->converterID); - mbState->converterID = 0; + if (mbState->converter != NULL) + ucnv_close((UConverter*)mbState->converter); + memset(mbState, 0, sizeof(mbstate_t)); return B_OK; } diff --git a/src/system/libroot/add-ons/icu/ICUThreadLocalStorageValue.cpp b/src/system/libroot/add-ons/icu/ICUThreadLocalStorageValue.cpp index d88410b6d7..e671f29667 100644 --- a/src/system/libroot/add-ons/icu/ICUThreadLocalStorageValue.cpp +++ b/src/system/libroot/add-ons/icu/ICUThreadLocalStorageValue.cpp @@ -8,7 +8,7 @@ #include -#include "ICUConverterManager.h" +#include namespace BPrivate { @@ -16,15 +16,16 @@ namespace Libroot { ICUThreadLocalStorageValue::ICUThreadLocalStorageValue() - : converterID(0) + : converter(NULL) { + charset[0] = '\0'; } ICUThreadLocalStorageValue::~ICUThreadLocalStorageValue() { - if (converterID != 0) - ICUConverterManager::Instance()->DropConverter(converterID); + if (converter != NULL) + ucnv_close(converter); } diff --git a/src/system/libroot/add-ons/icu/Jamfile b/src/system/libroot/add-ons/icu/Jamfile index 5640a26c29..762723658d 100644 --- a/src/system/libroot/add-ons/icu/Jamfile +++ b/src/system/libroot/add-ons/icu/Jamfile @@ -11,7 +11,6 @@ UsePrivateHeaders local sources = ICUCategoryData.cpp ICUCollateData.cpp - ICUConverterManager.cpp ICUCtypeData.cpp ICULocaleBackend.cpp ICULocaleconvData.cpp