From ebd3bcdb9be2d6a57fc5b3270dcb49a9e1894d11 Mon Sep 17 00:00:00 2001 From: John Scipione Date: Wed, 12 Feb 2014 20:15:18 -0500 Subject: [PATCH] exfat: handle 4-byte UTF-16 surrogate pairs ... in filenames. Replace the existing Unicode conversion functions with UTF conversion functions from js that he relicensed MIT for us. Put the UTF conversion functions in a private but shared code location so that they can be accessed throughout the kernel. Right now we only provide functions to convert between UTF-8 and UTF-16. At some point we should also add functions to convert between UTF-8 and UTF-32 and UTF-16 and UTF-32 but these aren't needed by exfat. Remove the old Unicode conversion functions from exfat as they assumed UCS-2 characters and don't work with UTF-16 used by exfat. Rename most variables with the term length with code unit where code units are intended. The term length, when used, means length in bytes while code units represent either a full 2-byte UTF-16 character or half a 4-byte surrogate pair. --- headers/private/system/convertutf.h | 25 ++ .../file_systems/exfat/DirectoryIterator.cpp | 136 +++++++---- .../file_systems/exfat/DirectoryIterator.h | 12 +- src/add-ons/kernel/file_systems/exfat/Jamfile | 1 - .../kernel/file_systems/exfat/Utility.cpp | 11 +- .../kernel/file_systems/exfat/Volume.cpp | 3 +- .../kernel/file_systems/exfat/encodings.cpp | 219 ------------------ .../kernel/file_systems/exfat/encodings.h | 20 -- src/add-ons/kernel/file_systems/exfat/exfat.h | 4 +- .../file_systems/exfat/kernel_interface.cpp | 4 +- src/system/kernel/Jamfile | 1 + src/system/kernel/convertutf.cpp | 135 +++++++++++ 12 files changed, 267 insertions(+), 304 deletions(-) create mode 100755 headers/private/system/convertutf.h delete mode 100644 src/add-ons/kernel/file_systems/exfat/encodings.cpp delete mode 100644 src/add-ons/kernel/file_systems/exfat/encodings.h create mode 100755 src/system/kernel/convertutf.cpp diff --git a/headers/private/system/convertutf.h b/headers/private/system/convertutf.h new file mode 100755 index 0000000000..99403153a9 --- /dev/null +++ b/headers/private/system/convertutf.h @@ -0,0 +1,25 @@ +/* + * Copyright 2014 Jonathan Schleifer + * Copyright 2014 Haiku, Inc. All rights reserved. + * + * Distributed under the terms of the MIT License. + * + * Authors: + * Jonathan Schleifer, js@webkeks.org + * John Scipione, jscipione@gmail.com + */ +#ifndef CONVERT_UTF_H +#define CONVERT_UTF_H + + +#include + + +ssize_t utf16le_to_utf8(const uint16* source, size_t sourceCodeUnitCount, + char* target, size_t targetLength); + +ssize_t utf16be_to_utf8(const uint16* source, size_t sourceCodeUnitCount, + char* target, size_t targetLength); + + +#endif // CONVERT_UTF_H diff --git a/src/add-ons/kernel/file_systems/exfat/DirectoryIterator.cpp b/src/add-ons/kernel/file_systems/exfat/DirectoryIterator.cpp index 10c5ee3e4d..278af03b1e 100644 --- a/src/add-ons/kernel/file_systems/exfat/DirectoryIterator.cpp +++ b/src/add-ons/kernel/file_systems/exfat/DirectoryIterator.cpp @@ -1,12 +1,21 @@ /* * Copyright 2011, Jérôme Duval, korli@users.berlios.de. - * This file may be used under the terms of the MIT License. + * Copyright 2014 Haiku, Inc. All rights reserved. + * + * Distributed under the terms of the MIT License. + * + * Authors: + * Jérôme Duval, korli@users.berlios.de + * John Scipione, jscipione@gmail.com */ #include "DirectoryIterator.h" -#include "encodings.h" +#include + +#include "convertutf.h" + #include "Inode.h" @@ -16,7 +25,11 @@ #else # define TRACE(x...) ; #endif -# define ERROR(x...) dprintf("\33[34mexfat:\33[0m " x) + +#define ERROR(x...) dprintf("\33[34mexfat:\33[0m " x) + + +// #pragma mark - DirectoryIterator DirectoryIterator::DirectoryIterator(Inode* inode) @@ -49,37 +62,55 @@ DirectoryIterator::GetNext(char* name, size_t* _nameLength, ino_t* _id, { if (fCluster == EXFAT_CLUSTER_END) return B_ENTRY_NOT_FOUND; + if (fOffset == -2) { if (*_nameLength < 3) return B_BUFFER_OVERFLOW; + *_nameLength = 2; strlcpy(name, "..", *_nameLength + 1); if (fInode->ID() == 1) *_id = fInode->ID(); else *_id = fInode->Parent(); + fOffset = -1; - TRACE("DirectoryIterator::GetNext() found ..\n"); + TRACE("DirectoryIterator::GetNext() found \"..\"\n"); + return B_OK; } else if (fOffset == -1) { if (*_nameLength < 2) return B_BUFFER_OVERFLOW; + *_nameLength = 1; strlcpy(name, ".", *_nameLength + 1); *_id = fInode->ID(); fOffset = 0; - TRACE("DirectoryIterator::GetNext() found .\n"); + TRACE("DirectoryIterator::GetNext() found \".\"\n"); + return B_OK; } - uchar unicodeName[EXFAT_FILENAME_MAX_LENGTH + 1]; - size_t nameLength = EXFAT_FILENAME_MAX_LENGTH; - status_t status = _GetNext(unicodeName, &nameLength, _id, visitor); - if (status == B_OK && name != NULL) { - status = unicode_to_utf8(unicodeName, nameLength, (uint8 *)name, - _nameLength); - TRACE("DirectoryIterator::GetNext() %" B_PRIu32 " %s, %" B_PRIdINO "\n", - fInode->Cluster(), name, *_id); + size_t utf16CodeUnitCount = EXFAT_FILENAME_MAX_LENGTH / sizeof(uint16); + uint16 utf16Name[utf16CodeUnitCount]; + status_t status = _GetNext(utf16Name, &utf16CodeUnitCount, _id, visitor); + if (status == B_OK && utf16CodeUnitCount > 0) { + ssize_t lengthOrStatus = utf16le_to_utf8(utf16Name, utf16CodeUnitCount, + name, *_nameLength); + if (lengthOrStatus < 0) { + status = (status_t)lengthOrStatus; + if (status == B_NAME_TOO_LONG) + *_nameLength = strlen(name); + } else + *_nameLength = (size_t)lengthOrStatus; + } + + if (status == B_OK) { + TRACE("DirectoryIterator::GetNext() cluster: %" B_PRIu32 " id: " + "%" B_PRIdINO " name: \"%s\", length: %zu\n", fInode->Cluster(), + *_id, name, *_nameLength); + } else if (status != B_ENTRY_NOT_FOUND) { + ERROR("DirectoryIterator::GetNext() (%s)\n", strerror(status)); } return status; @@ -97,26 +128,26 @@ DirectoryIterator::Lookup(const char* name, size_t nameLength, ino_t* _id) *_id = fInode->ID(); else *_id = fInode->Parent(); + return B_OK; } Rewind(); fOffset = 0; - uchar currentName[EXFAT_FILENAME_MAX_LENGTH + 1]; - size_t currentLength = EXFAT_FILENAME_MAX_LENGTH; - while (_GetNext((uchar*)currentName, ¤tLength, _id) == B_OK) { - char utfName[EXFAT_FILENAME_MAX_LENGTH]; - size_t utfLength = EXFAT_FILENAME_MAX_LENGTH; - unicode_to_utf8(currentName, currentLength, (uint8*)utfName, - &utfLength); - if (nameLength == utfLength - && strncmp(utfName, name, nameLength) == 0) { + size_t utf16CodeUnitCount = EXFAT_FILENAME_MAX_LENGTH / sizeof(uint16); + uint16 utf16Name[utf16CodeUnitCount]; + while (_GetNext(utf16Name, &utf16CodeUnitCount, _id) == B_OK) { + char utf8Name[nameLength + 1]; + ssize_t lengthOrStatus = utf16le_to_utf8(utf16Name, utf16CodeUnitCount, + utf8Name, sizeof(utf8Name)); + if (lengthOrStatus > 0 && (size_t)lengthOrStatus == nameLength + && strncmp(utf8Name, name, nameLength) == 0) { TRACE("DirectoryIterator::Lookup() found ID %" B_PRIdINO "\n", *_id); return B_OK; } - currentLength = EXFAT_FILENAME_MAX_LENGTH; + utf16CodeUnitCount = EXFAT_FILENAME_MAX_LENGTH / sizeof(uint16); } TRACE("DirectoryIterator::Lookup() not found %s\n", name); @@ -131,9 +162,9 @@ DirectoryIterator::LookupEntry(EntryVisitor* visitor) fCluster = fInode->Cluster(); fOffset = fInode->Offset(); - uchar unicodeName[EXFAT_FILENAME_MAX_LENGTH + 1]; - size_t nameLength = EXFAT_FILENAME_MAX_LENGTH; - return _GetNext(unicodeName, &nameLength, NULL, visitor); + size_t utf16CodeUnitCount = EXFAT_FILENAME_MAX_LENGTH / sizeof(uint16); + uint16 utf16Name[utf16CodeUnitCount]; + return _GetNext(utf16Name, &utf16CodeUnitCount, NULL, visitor); } @@ -151,7 +182,7 @@ DirectoryIterator::Iterate(EntryVisitor &visitor) { fOffset = 0; fCluster = fInode->StartCluster(); - + while (_NextEntry() != B_ENTRY_NOT_FOUND) { switch (fCurrent->type) { case EXFAT_ENTRY_TYPE_BITMAP: @@ -178,13 +209,14 @@ DirectoryIterator::Iterate(EntryVisitor &visitor) status_t -DirectoryIterator::_GetNext(uchar* name, size_t* _nameLength, ino_t* _id, - EntryVisitor* visitor) +DirectoryIterator::_GetNext(uint16* utf16Name, size_t* _codeUnitCount, + ino_t* _id, EntryVisitor* visitor) { - size_t nameMax = *_nameLength; + size_t nameMax = *_codeUnitCount; size_t nameIndex = 0; status_t status; int32 chunkCount = 1; + while ((status = _NextEntry()) == B_OK) { TRACE("DirectoryIterator::_GetNext() %" B_PRIu32 "/%p, type 0x%x, " "offset %" B_PRId64 "\n", fInode->Cluster(), fCurrent, @@ -195,24 +227,24 @@ DirectoryIterator::_GetNext(uchar* name, size_t* _nameLength, ino_t* _id, *_id = fInode->GetVolume()->GetIno(fCluster, fOffset - 1, fInode->ID()); } - if (visitor != NULL) - visitor->VisitFile(fCurrent); TRACE("DirectoryIterator::_GetNext() File chunkCount %" B_PRId32 "\n", chunkCount); + if (visitor != NULL) + visitor->VisitFile(fCurrent); } else if (fCurrent->type == EXFAT_ENTRY_TYPE_FILEINFO) { chunkCount--; - TRACE("DirectoryIterator::_GetNext() Filename length %d\n", - fCurrent->file_info.name_length); - *_nameLength = fCurrent->file_info.name_length * 2; + *_codeUnitCount = (size_t)fCurrent->file_info.name_length; + TRACE("DirectoryIterator::_GetNext() Filename chunk: %" B_PRId32 + ", code unit count: %" B_PRIu8 "\n", chunkCount, *_codeUnitCount); if (visitor != NULL) visitor->VisitFileInfo(fCurrent); } else if (fCurrent->type == EXFAT_ENTRY_TYPE_FILENAME) { - TRACE("DirectoryIterator::_GetNext() Filename\n"); - memcpy((uint8*)name + nameIndex, fCurrent->file_name.name, - sizeof(fCurrent->file_name.name)); - nameIndex += sizeof(fCurrent->file_name.name); - name[nameIndex] = '\0'; chunkCount--; + size_t utf16Length = sizeof(fCurrent->file_name.name); + memcpy(utf16Name + nameIndex, fCurrent->file_name.name, utf16Length); + nameIndex += utf16Length / sizeof(uint16); + TRACE("DirectoryIterator::_GetNext() Filename index: %zu\n", + nameIndex); if (visitor != NULL) visitor->VisitFilename(fCurrent); } @@ -221,16 +253,18 @@ DirectoryIterator::_GetNext(uchar* name, size_t* _nameLength, ino_t* _id, break; } - if (status == B_OK) { - //*_nameLength = nameIndex; #ifdef TRACE_EXFAT - char utfName[EXFAT_FILENAME_MAX_LENGTH]; - size_t utfLen = EXFAT_FILENAME_MAX_LENGTH; - unicode_to_utf8(name, nameIndex, (uint8*)utfName, &utfLen); - TRACE("DirectoryIterator::_GetNext() Found %s %ld\n", utfName, - *_nameLength); -#endif + if (status == B_OK) { + size_t utf8Length = B_FILE_NAME_LENGTH * 4; + char utf8Name[utf8Length + 1]; + ssize_t length = utf16le_to_utf8(utf16Name, *_codeUnitCount, utf8Name, + utf8Length); + if (length > 0) { + TRACE("DirectoryIterator::_GetNext() found name: \"%s\", " + "length: %d\n", utf8Name, length); + } } +#endif return status; } @@ -248,20 +282,22 @@ DirectoryIterator::_NextEntry() block); fCurrent = (struct exfat_entry*)fBlock.SetTo(block) + fOffset % fInode->GetVolume()->EntriesPerBlock(); - } else if ((fOffset % fInode->GetVolume()->EntriesPerBlock()) == 0) { + } else if ((fOffset % fInode->GetVolume()->EntriesPerBlock()) == 0) { fsblock_t block; if ((fOffset % fInode->GetVolume()->EntriesPerCluster()) == 0) { fCluster = fInode->NextCluster(fCluster); if (fCluster == EXFAT_CLUSTER_END) return B_ENTRY_NOT_FOUND; + fInode->GetVolume()->ClusterToBlock(fCluster, block); } else block = fBlock.BlockNumber() + 1; + TRACE("DirectoryIterator::_NextEntry() block %" B_PRIu64 "\n", block); fCurrent = (struct exfat_entry*)fBlock.SetTo(block); } else fCurrent++; - fOffset++; + fOffset++; return fCurrent->type == 0 ? B_ENTRY_NOT_FOUND : B_OK; } diff --git a/src/add-ons/kernel/file_systems/exfat/DirectoryIterator.h b/src/add-ons/kernel/file_systems/exfat/DirectoryIterator.h index 27b7dd3deb..daa3ea53a7 100644 --- a/src/add-ons/kernel/file_systems/exfat/DirectoryIterator.h +++ b/src/add-ons/kernel/file_systems/exfat/DirectoryIterator.h @@ -1,6 +1,12 @@ /* * Copyright 2011, Jérôme Duval, korli@users.berlios.de. - * This file may be used under the terms of the MIT License. + * Copyright 2014 Haiku, Inc. All rights reserved. + * + * Distributed under the terms of the MIT License. + * + * Authors: + * Jérôme Duval, korli@users.berlios.de + * John Scipione, jscipione@gmail.com */ #ifndef DIRECTORYITERATOR_H #define DIRECTORYITERATOR_H @@ -46,8 +52,8 @@ public: void Iterate(EntryVisitor &visitor); private: - status_t _GetNext(uchar* unicodename, - size_t* _nameLength, ino_t* _id, + status_t _GetNext(uint16* unicodeName, + size_t* _codeUnitCount, ino_t* _id, EntryVisitor* visitor = NULL); status_t _NextEntry(); diff --git a/src/add-ons/kernel/file_systems/exfat/Jamfile b/src/add-ons/kernel/file_systems/exfat/Jamfile index d318694523..cdf366f401 100644 --- a/src/add-ons/kernel/file_systems/exfat/Jamfile +++ b/src/add-ons/kernel/file_systems/exfat/Jamfile @@ -7,7 +7,6 @@ UsePrivateKernelHeaders ; KernelAddon exfat : DataStream.cpp DirectoryIterator.cpp - encodings.cpp Inode.cpp kernel_interface.cpp Utility.cpp diff --git a/src/add-ons/kernel/file_systems/exfat/Utility.cpp b/src/add-ons/kernel/file_systems/exfat/Utility.cpp index 238531724a..98f6104fe9 100644 --- a/src/add-ons/kernel/file_systems/exfat/Utility.cpp +++ b/src/add-ons/kernel/file_systems/exfat/Utility.cpp @@ -18,7 +18,7 @@ #include -#include "encodings.h" +#include "convertutf.h" status_t @@ -30,11 +30,10 @@ get_volume_name(struct exfat_entry* entry, char* name, size_t length) if (entry->type == EXFAT_ENTRY_TYPE_NOT_IN_USE) name = ""; else if (entry->type == EXFAT_ENTRY_TYPE_LABEL) { - status_t result - = unicode_to_utf8((const uchar*)entry->volume_label.name, - entry->volume_label.length * 2, (uint8*)name, &length); - if (result != B_OK) - return result; + ssize_t utf8Length = utf16le_to_utf8(entry->volume_label.name, + entry->volume_label.length, name, length); + if (utf8Length < 0) + return (status_t)utf8Length; } else return B_NAME_NOT_FOUND; diff --git a/src/add-ons/kernel/file_systems/exfat/Volume.cpp b/src/add-ons/kernel/file_systems/exfat/Volume.cpp index c30257078f..1e885045f3 100644 --- a/src/add-ons/kernel/file_systems/exfat/Volume.cpp +++ b/src/add-ons/kernel/file_systems/exfat/Volume.cpp @@ -339,7 +339,8 @@ Volume::Mount(const char* deviceName, uint32 flags) if (status != B_OK) return status; - off_t partitionSize = (off_t)fSuperBlock.NumBlocks() << fSuperBlock.BlockShift(); + off_t partitionSize = (off_t)fSuperBlock.NumBlocks() + << fSuperBlock.BlockShift(); if (deviceSize < partitionSize) return B_BAD_VALUE; diff --git a/src/add-ons/kernel/file_systems/exfat/encodings.cpp b/src/add-ons/kernel/file_systems/exfat/encodings.cpp deleted file mode 100644 index ac844dfc43..0000000000 --- a/src/add-ons/kernel/file_systems/exfat/encodings.cpp +++ /dev/null @@ -1,219 +0,0 @@ -/* - Copyright 1999-2001, Be Incorporated. All Rights Reserved. - This file may be used under the terms of the Be Sample Code License. -*/ - - -#include -#include -#include -#include -#include -#include - - -#include "encodings.h" - - -// Pierre's Uber Macro -#define u_lendian_to_utf8(str, uni_str)\ -{\ - if ((B_LENDIAN_TO_HOST_INT16(uni_str[0])&0xff80) == 0)\ - *str++ = B_LENDIAN_TO_HOST_INT16(*uni_str++);\ - else if ((B_LENDIAN_TO_HOST_INT16(uni_str[0])&0xf800) == 0) {\ - str[0] = 0xc0|(B_LENDIAN_TO_HOST_INT16(uni_str[0])>>6);\ - str[1] = 0x80|(B_LENDIAN_TO_HOST_INT16(*uni_str++)&0x3f);\ - str += 2;\ - } else if ((B_LENDIAN_TO_HOST_INT16(uni_str[0])&0xfc00) != 0xd800) {\ - str[0] = 0xe0|(B_LENDIAN_TO_HOST_INT16(uni_str[0])>>12);\ - str[1] = 0x80|((B_LENDIAN_TO_HOST_INT16(uni_str[0])>>6)&0x3f);\ - str[2] = 0x80|(B_LENDIAN_TO_HOST_INT16(*uni_str++)&0x3f);\ - str += 3;\ - } else {\ - int val;\ - val = ((B_LENDIAN_TO_HOST_INT16(uni_str[0])-0xd7c0)<<10) | (B_LENDIAN_TO_HOST_INT16(uni_str[1])&0x3ff);\ - str[0] = 0xf0 | (val>>18);\ - str[1] = 0x80 | ((val>>12)&0x3f);\ - str[2] = 0x80 | ((val>>6)&0x3f);\ - str[3] = 0x80 | (val&0x3f);\ - uni_str += 2; str += 4;\ - }\ -} - -// Pierre's Uber Macro -#define u_hostendian_to_utf8(str, uni_str)\ -{\ - if ((uni_str[0]&0xff80) == 0)\ - *str++ = *uni_str++;\ - else if ((uni_str[0]&0xf800) == 0) {\ - str[0] = 0xc0|(uni_str[0]>>6);\ - str[1] = 0x80|(*uni_str++&0x3f);\ - str += 2;\ - } else if ((uni_str[0]&0xfc00) != 0xd800) {\ - str[0] = 0xe0|(uni_str[0]>>12);\ - str[1] = 0x80|((uni_str[0]>>6)&0x3f);\ - str[2] = 0x80|(*uni_str++&0x3f);\ - str += 3;\ - } else {\ - int val;\ - val = ((uni_str[0]-0xd7c0)<<10) | (uni_str[1]&0x3ff);\ - str[0] = 0xf0 | (val>>18);\ - str[1] = 0x80 | ((val>>12)&0x3f);\ - str[2] = 0x80 | ((val>>6)&0x3f);\ - str[3] = 0x80 | (val&0x3f);\ - uni_str += 2; str += 4;\ - }\ -} - -// Another Uber Macro -#define utf8_to_u_hostendian(str, uni_str, err_flag) \ -{\ - err_flag = 0;\ - if ((str[0]&0x80) == 0)\ - *uni_str++ = *str++;\ - else if ((str[1] & 0xC0) != 0x80) {\ - *uni_str++ = 0xfffd;\ - str+=1;\ - } else if ((str[0]&0x20) == 0) {\ - *uni_str++ = ((str[0]&31)<<6) | (str[1]&63);\ - str+=2;\ - } else if ((str[2] & 0xC0) != 0x80) {\ - *uni_str++ = 0xfffd;\ - str+=2;\ - } else if ((str[0]&0x10) == 0) {\ - *uni_str++ = ((str[0]&15)<<12) | ((str[1]&63)<<6) | (str[2]&63);\ - str+=3;\ - } else if ((str[3] & 0xC0) != 0x80) {\ - *uni_str++ = 0xfffd;\ - str+=3;\ - } else {\ - err_flag = 1;\ - }\ -} - -// Count the number of bytes of a UTF-8 character -#define utf8_char_len(c) ((((int32)0xE5000000 >> ((c >> 3) & 0x1E)) & 3) + 1) - -// converts LENDIAN unicode to utf8 -static status_t -_lendian_unicode_to_utf8( - const char *src, - size_t *srcLen, - char *dst, - size_t *dstLen) -{ - size_t srcLimit = *srcLen; - size_t dstLimit = *dstLen; - size_t srcCount = 0; - size_t dstCount = 0; - status_t status = B_ERROR; - - for (srcCount = 0; srcCount < srcLimit; srcCount += 2) { - uint16 *UNICODE = (uint16 *)&src[srcCount]; - if (*UNICODE == 0) - break; - uchar utf8[4]; - uchar *UTF8 = utf8; - int32 utf8Len; - int32 j; - - u_lendian_to_utf8(UTF8, UNICODE); - - utf8Len = UTF8 - utf8; - if ((dstCount + utf8Len) > dstLimit) { - status = B_BUFFER_OVERFLOW; - break; - } - - for (j = 0; j < utf8Len; j++) - dst[dstCount + j] = utf8[j]; - dstCount += utf8Len; - status = B_OK; - } - - *srcLen = srcCount; - *dstLen = dstCount; - dst[dstCount] = '\0'; - - return status; -} - -// utf8 to LENDIAN unicode -static status_t -_utf8_to_lendian_unicode( - const char *src, - size_t *srcLen, - char *dst, - size_t *dstLen) -{ - size_t srcLimit = *srcLen; - size_t dstLimit = *dstLen - 1; - size_t srcCount = 0; - size_t dstCount = 0; - status_t status = B_ERROR; - - while ((srcCount < srcLimit) && (dstCount < dstLimit)) { - uint16 unicode; - uint16 *UNICODE = &unicode; - uchar *UTF8 = (uchar *)src + srcCount; - int err_flag; - - if ((srcCount + utf8_char_len(src[srcCount])) > srcLimit) - break; - - utf8_to_u_hostendian(UTF8, UNICODE, err_flag); - if (err_flag == 1) - return EINVAL; - - unicode = B_HOST_TO_LENDIAN_INT16(unicode); - if ((dstCount + 1) > dstLimit) { - status = B_BUFFER_OVERFLOW; - break; - } - dst[dstCount++] = unicode & 0xFF; - dst[dstCount++] = unicode >> 8; - - srcCount += UTF8 - ((uchar *)(src + srcCount)); - status = B_OK; - } - - *srcLen = srcCount; - *dstLen = dstCount; - - return status; -} - - -// takes a unicode name of unilen uchar's and converts to a utf8 name of at -// most utf8len uint8's -status_t unicode_to_utf8(const uchar *uni, size_t unilen, uint8 *utf8, - size_t *utf8len) -{ - //size_t origlen = unilen; - status_t result = _lendian_unicode_to_utf8((char *)uni, - &unilen, (char *)utf8, utf8len); - - /*if (unilen < origlen) { - panic("Name is too long (%lx < %lx)\n", unilen, origlen); - return B_ERROR; - }*/ - - return result; -} - - -status_t utf8_to_unicode(const char *utf8, uchar *uni, size_t *unilen) -{ - size_t origlen = strlen(utf8) + 1; - size_t utf8len = origlen; - - status_t result = _utf8_to_lendian_unicode(utf8, - &utf8len, (char *)uni, unilen); - - /*if (origlen < utf8len) { - panic("Name is too long (%lx < %lx)\n", *unilen, origlen); - return B_ERROR; - }*/ - - return result; -} diff --git a/src/add-ons/kernel/file_systems/exfat/encodings.h b/src/add-ons/kernel/file_systems/exfat/encodings.h deleted file mode 100644 index eaddd918bd..0000000000 --- a/src/add-ons/kernel/file_systems/exfat/encodings.h +++ /dev/null @@ -1,20 +0,0 @@ -/* - Copyright 1999-2001, Be Incorporated. All Rights Reserved. - This file may be used under the terms of the Be Sample Code License. -*/ -#ifndef _ENCODINGS_H_ -#define _ENCODINGS_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -status_t unicode_to_utf8(const uchar *uni, size_t unilen, uint8 *utf8, - size_t *utf8len); -status_t utf8_to_unicode(const char *utf8, uchar *uni, size_t *unilen); - -#ifdef __cplusplus -} -#endif - -#endif // _ENCODINGS_H_ diff --git a/src/add-ons/kernel/file_systems/exfat/exfat.h b/src/add-ons/kernel/file_systems/exfat/exfat.h index 8a705960fa..b08a715018 100644 --- a/src/add-ons/kernel/file_systems/exfat/exfat.h +++ b/src/add-ons/kernel/file_systems/exfat/exfat.h @@ -92,9 +92,9 @@ struct exfat_super_block { #define EXFAT_CLUSTER_END 0xffffffff #define EXFAT_ENTRY_ATTRIB_SUBDIR 0x10 -#define EXFAT_ENTRY_FLAG_CONTIGUOUS 0x3 +#define EXFAT_ENTRY_FLAG_CONTIGUOUS 0x3 -#define EXFAT_FILENAME_MAX_LENGTH 512 +#define EXFAT_FILENAME_MAX_LENGTH 512 struct exfat_entry { diff --git a/src/add-ons/kernel/file_systems/exfat/kernel_interface.cpp b/src/add-ons/kernel/file_systems/exfat/kernel_interface.cpp index dcb42430e6..627a38bf22 100644 --- a/src/add-ons/kernel/file_systems/exfat/kernel_interface.cpp +++ b/src/add-ons/kernel/file_systems/exfat/kernel_interface.cpp @@ -401,7 +401,7 @@ exfat_lookup(fs_volume* _volume, fs_vnode* _directory, const char* name, } TRACE("exfat_lookup: ID %" B_PRIdINO "\n", *_vnodeID); - + return get_vnode(volume->FSVolume(), *_vnodeID, NULL); } @@ -570,7 +570,7 @@ exfat_read_dir(fs_volume *_volume, fs_vnode *_node, void *_cookie, TRACE("exfat_read_dir\n"); DirectoryIterator* iterator = (DirectoryIterator*)_cookie; Volume* volume = (Volume*)_volume->private_volume; - + uint32 maxCount = *_num; uint32 count = 0; diff --git a/src/system/kernel/Jamfile b/src/system/kernel/Jamfile index 677258d6d0..8a4fd0f436 100644 --- a/src/system/kernel/Jamfile +++ b/src/system/kernel/Jamfile @@ -27,6 +27,7 @@ KernelMergeObject kernel_core.o : boot_splash.cpp commpage.cpp condition_variable.cpp + convertutf.cpp cpu.cpp DPC.cpp elf.cpp diff --git a/src/system/kernel/convertutf.cpp b/src/system/kernel/convertutf.cpp new file mode 100755 index 0000000000..897e580dee --- /dev/null +++ b/src/system/kernel/convertutf.cpp @@ -0,0 +1,135 @@ +/* + * Copyright 2014 Jonathan Schleifer + * Copyright 2014 Haiku, Inc. All rights reserved. + * + * Distributed under the terms of the MIT License. + * + * Authors: + * Jonathan Schleifer, js@webkeks.org + * John Scipione, jscipione@gmail.com + */ + + +#include "convertutf.h" + + +#include +#include +#include + + +static inline size_t +glyph_length(uint32 glyph) +{ + if (glyph < 0x80) + return 1; + else if (glyph < 0x800) + return 2; + else if (glyph < 0x10000) + return 3; + else if (glyph < 0x110000) + return 4; + + return 0; +} + + +void +encode_glyph(uint32 glyph, size_t glyphLength, char* buffer) +{ + if (glyphLength == 1) { + *buffer = glyph; + } else if (glyphLength == 2) { + *buffer++ = 0xC0 | (glyph >> 6); + *buffer = 0x80 | (glyph & 0x3F); + } else if (glyphLength == 3) { + *buffer++ = 0xE0 | (glyph >> 12); + *buffer++ = 0x80 | (glyph >> 6 & 0x3F); + *buffer = 0x80 | (glyph & 0x3F); + } else if (glyphLength == 4) { + *buffer++ = 0xF0 | (glyph >> 18); + *buffer++ = 0x80 | (glyph >> 12 & 0x3F); + *buffer++ = 0x80 | (glyph >> 6 & 0x3F); + *buffer = 0x80 | (glyph & 0x3F); + } +} + + +static ssize_t +utf16_to_utf8(const uint16* source, size_t sourceCodeUnitCount, char* target, + size_t targetLength, bool isLittleEndian) +{ + if (source == NULL || sourceCodeUnitCount == 0 + || target == NULL || targetLength == 0) { + return B_BAD_VALUE; + } + + ssize_t outLength = 0; + + for (size_t i = 0; i < sourceCodeUnitCount; i++) { + uint32 glyph = isLittleEndian + ? B_LENDIAN_TO_HOST_INT32(source[i]) + : B_BENDIAN_TO_HOST_INT32(source[i]); + + if ((glyph & 0xFC00) == 0xDC00) { + // missing high surrogate + return B_BAD_VALUE; + } + + if ((glyph & 0xFC00) == 0xD800) { + if (sourceCodeUnitCount <= i + 1) { + // high surrogate at end of string + return B_BAD_VALUE; + } + + uint32 low = isLittleEndian + ? B_LENDIAN_TO_HOST_INT32(source[i + 1]) + : B_BENDIAN_TO_HOST_INT32(source[i + 1]); + if ((low & 0xFC00) != 0xDC00) { + // missing low surrogate + return B_BAD_VALUE; + } + + glyph = (((glyph & 0x3FF) << 10) | (low & 0x3FF)) + 0x10000; + i++; + } + + size_t glyphLength = glyph_length(glyph); + if (glyphLength == 0) + return B_BAD_VALUE; + else if (outLength + glyphLength >= targetLength + || outLength + glyphLength >= B_FILE_NAME_LENGTH) { + // NUL terminate the string so the caller can use the + // abbreviated version in this case. Since the length + // isn't returned the caller will need to call strlen() + // to get the length of the string. + target[outLength] = '\0'; + return B_NAME_TOO_LONG; + } + + encode_glyph(glyph, glyphLength, target + outLength); + outLength += glyphLength; + } + + target[outLength] = '\0'; + + return outLength; +} + + +ssize_t +utf16le_to_utf8(const uint16* source, size_t sourceCodeUnitCount, + char* target, size_t targetLength) +{ + return utf16_to_utf8(source, sourceCodeUnitCount, target, targetLength, + true); +} + + +ssize_t +utf16be_to_utf8(const uint16* source, size_t sourceCodeUnitCount, + char* target, size_t targetLength) +{ + return utf16_to_utf8(source, sourceCodeUnitCount, target, targetLength, + false); +}