diff --git a/headers/os/locale/TextEncoding.h b/headers/os/locale/TextEncoding.h new file mode 100644 index 0000000000..635a8e81b1 --- /dev/null +++ b/headers/os/locale/TextEncoding.h @@ -0,0 +1,28 @@ +/* + * Copyright 2016, Haiku, inc. + * Distributed under terms of the MIT license. + */ + + +#ifndef TEXTENCODING_H +#define TEXTENCODING_H + + +#include + +#include + + +class TextEncoding +{ + public: + TextEncoding(const char* data, size_t length); + + BString GetName(); + + private: + BString fName; +}; + + +#endif /* !TEXTENCODING_H */ diff --git a/src/add-ons/translators/stxt/STXTTranslator.cpp b/src/add-ons/translators/stxt/STXTTranslator.cpp index 77b500e6f9..d1da5d8cf2 100644 --- a/src/add-ons/translators/stxt/STXTTranslator.cpp +++ b/src/add-ons/translators/stxt/STXTTranslator.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -32,7 +33,7 @@ using namespace std; #define B_TRANSLATION_CONTEXT "STXTTranslator" #define READ_BUFFER_SIZE 32768 -#define DATA_BUFFER_SIZE 256 +#define DATA_BUFFER_SIZE 2048 // The input formats that this translator supports. static const translation_format sInputFormats[] = { @@ -116,666 +117,6 @@ make_nth_translator(int32 n, image_id you, uint32 flags, ...) } -// #pragma mark - ascmagic.c from the BSD file tool -/* - * The following code has been taken from version 4.17 of the BSD file tool, - * file ascmagic.c, modified for our purpose. - */ - -/* - * Copyright (c) Ian F. Darwin 1986-1995. - * Software written by Ian F. Darwin and others; - * maintained 1995-present by Christos Zoulas and others. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice immediately at the beginning of the file, without modification, - * this list of conditions, and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -/* - * ASCII magic -- file types that we know based on keywords - * that can appear anywhere in the file. - * bool found = false; - if (subtypeMimeSpecific != NULL) { - mimeType->SetTo(subtypeMimeSpecific); - if (mimeType->IsInstalled()) - found = true; - } - if (!found && subtypeMimeGeneric != NULL) { - mimeType->SetTo(subtypeMimeGeneric); - if (mimeType->IsInstalled()) - found = true; - } - if (!found) - mimeType->SetTo("text/plain"); - - * Extensively modified by Eric Fischer in July, 2000, - * to handle character codes other than ASCII on a unified basis. - * - * Joerg Wunsch wrote the original support for 8-bit - * international characters, now subsumed into this file. - */ - -#include -#include -#include -#include -#include -#include -#include "names.h" - -typedef unsigned long my_unichar; - -#define MAXLINELEN 300 /* longest sane line length */ -#define ISSPC(x) ((x) == ' ' || (x) == '\t' || (x) == '\r' || (x) == '\n' \ - || (x) == 0x85 || (x) == '\f') - -static int looks_ascii(const unsigned char *, size_t, my_unichar *, size_t *); -static int looks_utf8(const unsigned char *, size_t, my_unichar *, size_t *); -static int looks_unicode(const unsigned char *, size_t, my_unichar *, size_t *); -static int looks_latin1(const unsigned char *, size_t, my_unichar *, size_t *); -static int looks_extended(const unsigned char *, size_t, my_unichar *, size_t *); -static void from_ebcdic(const unsigned char *, size_t, unsigned char *); -static int ascmatch(const unsigned char *, const my_unichar *, size_t); - - -static int -file_ascmagic(const unsigned char *buf, size_t nbytes, BMimeType* mimeType, - const char*& encoding) -{ - size_t i; - unsigned char *nbuf = NULL; - my_unichar *ubuf = NULL; - size_t ulen; - struct names *p; - int rv = -1; - - const char *code = NULL; - encoding = NULL; - const char *type = NULL; - const char *subtype = NULL; - const char *subtypeMimeGeneric = NULL; - const char *subtypeMimeSpecific = NULL; - - int has_escapes = 0; - int has_backspace = 0; - int seen_cr = 0; - - int n_crlf = 0; - int n_lf = 0; - int n_cr = 0; - int n_nel = 0; - - int last_line_end = -1; - int has_long_lines = 0; - - if ((nbuf = (unsigned char*)malloc((nbytes + 1) * sizeof(nbuf[0]))) == NULL) - goto done; - if ((ubuf = (my_unichar*)malloc((nbytes + 1) * sizeof(ubuf[0]))) == NULL) - goto done; - - /* - * Then try to determine whether it's any character code we can - * identify. Each of these tests, if it succeeds, will leave - * the text converted into one-my_unichar-per-character Unicode in - * ubuf, and the number of characters converted in ulen. - */ - if (nbytes == 0) { - code = "UTF-8 Unicode"; - encoding = NULL; // "UTF-8"; - type = "text"; - rv = 1; - } else if (looks_ascii(buf, nbytes, ubuf, &ulen)) { - code = "ASCII"; - encoding = NULL; //"us-ascii"; - type = "text"; - if (nbytes == 1) { - // no further tests - rv = 1; - } - } else if (looks_utf8(buf, nbytes, ubuf, &ulen)) { - code = "UTF-8 Unicode"; - encoding = NULL; // "UTF-8"; - type = "text"; - } else if ((i = looks_unicode(buf, nbytes, ubuf, &ulen)) != 0) { - if (i == 1) { - code = "Little-endian UTF-16 Unicode"; - encoding = "UTF-16"; - } else { - code = "Big-endian UTF-16 Unicode"; - encoding = "UTF-16"; - } - - type = "character data"; - } else if (looks_latin1(buf, nbytes, ubuf, &ulen)) { - code = "ISO-8859"; - type = "text"; - encoding = "iso-8859-1"; - } else if (looks_extended(buf, nbytes, ubuf, &ulen)) { - code = "Non-ISO extended-ASCII"; - type = "text"; - encoding = "unknown"; - } else { - from_ebcdic(buf, nbytes, nbuf); - - if (looks_ascii(nbuf, nbytes, ubuf, &ulen)) { - code = "EBCDIC"; - type = "character data"; - encoding = "ebcdic"; - } else if (looks_latin1(nbuf, nbytes, ubuf, &ulen)) { - code = "International EBCDIC"; - type = "character data"; - encoding = "ebcdic"; - } else { - rv = 0; - goto done; /* doesn't look like text at all */ - } - } - - if (nbytes <= 1) { - if (rv == -1) - rv = 0; - goto done; - } - - /* - * for troff, look for . + letter + letter or .\"; - * this must be done to disambiguate tar archives' ./file - * and other trash from real troff input. - * - * I believe Plan 9 troff allows non-ASCII characters in the names - * of macros, so this test might possibly fail on such a file. - */ - if (*ubuf == '.') { - my_unichar *tp = ubuf + 1; - - while (ISSPC(*tp)) - ++tp; /* skip leading whitespace */ - if ((tp[0] == '\\' && tp[1] == '\"') || - (isascii((unsigned char)tp[0]) && - isalnum((unsigned char)tp[0]) && - isascii((unsigned char)tp[1]) && - isalnum((unsigned char)tp[1]) && - ISSPC(tp[2]))) { - subtypeMimeGeneric = "text/x-source-code"; - subtypeMimeSpecific = "text/troff"; - subtype = "troff or preprocessor input"; - goto subtype_identified; - } - } - - if ((*buf == 'c' || *buf == 'C') && ISSPC(buf[1])) { - subtypeMimeGeneric = "text/x-source-code"; - subtypeMimeSpecific = "text/fortran"; - subtype = "fortran program"; - goto subtype_identified; - } - - /* look for tokens from names.h - this is expensive! */ - - i = 0; - while (i < ulen) { - size_t end; - - /* - * skip past any leading space - */ - while (i < ulen && ISSPC(ubuf[i])) - i++; - if (i >= ulen) - break; - - /* - * find the next whitespace - */ - for (end = i + 1; end < nbytes; end++) - if (ISSPC(ubuf[end])) - break; - - /* - * compare the word thus isolated against the token list - */ - for (p = names; p < names + NNAMES; p++) { - if (ascmatch((const unsigned char *)p->name, ubuf + i, - end - i)) { - subtype = types[p->type].human; - subtypeMimeGeneric = types[p->type].generic_mime; - subtypeMimeSpecific = types[p->type].specific_mime; - goto subtype_identified; - } - } - - i = end; - } - -subtype_identified: - - /* - * Now try to discover other details about the file. - */ - for (i = 0; i < ulen; i++) { - if (ubuf[i] == '\n') { - if (seen_cr) - n_crlf++; - else - n_lf++; - last_line_end = i; - } else if (seen_cr) - n_cr++; - - seen_cr = (ubuf[i] == '\r'); - if (seen_cr) - last_line_end = i; - - if (ubuf[i] == 0x85) { /* X3.64/ECMA-43 "next line" character */ - n_nel++; - last_line_end = i; - } - - /* If this line is _longer_ than MAXLINELEN, remember it. */ - if ((int)i > last_line_end + MAXLINELEN) - has_long_lines = 1; - - if (ubuf[i] == '\033') - has_escapes = 1; - if (ubuf[i] == '\b') - has_backspace = 1; - } - - rv = 1; -done: - if (nbuf) - free(nbuf); - if (ubuf) - free(ubuf); - - if (rv) { - // If we have identified the subtype, return it, otherwise just - // text/plain. - - bool found = false; - if (subtypeMimeSpecific != NULL) { - mimeType->SetTo(subtypeMimeSpecific); - if (mimeType->IsInstalled()) - found = true; - } - if (!found && subtypeMimeGeneric != NULL) { - mimeType->SetTo(subtypeMimeGeneric); - if (mimeType->IsInstalled()) - found = true; - } - if (!found) - mimeType->SetTo("text/plain"); - } - - return rv; -} - -static int -ascmatch(const unsigned char *s, const my_unichar *us, size_t ulen) -{ - size_t i; - - for (i = 0; i < ulen; i++) { - if (s[i] != us[i]) - return 0; - } - - if (s[i]) - return 0; - else - return 1; -} - -/* - * This table reflects a particular philosophy about what constitutes - * "text," and there is room for disagreement about it. - * - * Version 3.31 of the file command considered a file to be ASCII if - * each of its characters was approved by either the isascii() or - * isalpha() function. On most systems, this would mean that any - * file consisting only of characters in the range 0x00 ... 0x7F - * would be called ASCII text, but many systems might reasonably - * consider some characters outside this range to be alphabetic, - * so the file command would call such characters ASCII. It might - * have been more accurate to call this "considered textual on the - * local system" than "ASCII." - * - * It considered a file to be "International language text" if each - * of its characters was either an ASCII printing character (according - * to the real ASCII standard, not the above test), a character in - * the range 0x80 ... 0xFF, or one of the following control characters: - * backspace, tab, line feed, vertical tab, form feed, carriage return, - * escape. No attempt was made to determine the language in which files - * of this type were written. - * - * - * The table below considers a file to be ASCII if all of its characters - * are either ASCII printing characters (again, according to the X3.4 - * standard, not isascii()) or any of the following controls: bell, - * backspace, tab, line feed, form feed, carriage return, esc, nextline. - * - * I include bell because some programs (particularly shell scripts) - * use it literally, even though it is rare in normal text. I exclude - * vertical tab because it never seems to be used in real text. I also - * include, with hesitation, the X3.64/ECMA-43 control nextline (0x85), - * because that's what the dd EBCDIC->ASCII table maps the EBCDIC newline - * character to. It might be more appropriate to include it in the 8859 - * set instead of the ASCII set, but it's got to be included in *something* - * we recognize or EBCDIC files aren't going to be considered textual. - * Some old Unix source files use SO/SI (^N/^O) to shift between Greek - * and Latin characters, so these should possibly be allowed. But they - * make a real mess on VT100-style displays if they're not paired properly, - * so we are probably better off not calling them text. - * - * A file is considered to be ISO-8859 text if its characters are all - * either ASCII, according to the above definition, or printing characters - * from the ISO-8859 8-bit extension, characters 0xA0 ... 0xFF. - * - * Finally, a file is considered to be international text from some other - * character code if its characters are all either ISO-8859 (according to - * the above definition) or characters in the range 0x80 ... 0x9F, which - * ISO-8859 considers to be control characters but the IBM PC and Macintosh - * consider to be printing characters. - */ - -#define F 0 /* character never appears in text */ -#define T 1 /* character appears in plain ASCII text */ -#define I 2 /* character appears in ISO-8859 text */ -#define X 3 /* character appears in non-ISO extended ASCII (Mac, IBM PC) */ - -static char text_chars[256] = { - /* BEL BS HT LF FF CR */ - F, F, F, F, F, F, F, T, T, T, T, F, T, T, F, F, /* 0x0X */ - /* ESC */ - F, F, F, F, F, F, F, F, F, F, F, T, F, F, F, F, /* 0x1X */ - T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x2X */ - T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x3X */ - T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x4X */ - T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x5X */ - T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x6X */ - T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, F, /* 0x7X */ - /* NEL */ - X, X, X, X, X, T, X, X, X, X, X, X, X, X, X, X, /* 0x8X */ - X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, /* 0x9X */ - I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xaX */ - I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xbX */ - I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xcX */ - I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xdX */ - I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xeX */ - I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I /* 0xfX */ -}; - -static int -looks_ascii(const unsigned char *buf, size_t nbytes, my_unichar *ubuf, - size_t *ulen) -{ - int i; - - *ulen = 0; - - for (i = 0; i < (int)nbytes; i++) { - int t = text_chars[buf[i]]; - - if (t != T) - return 0; - - ubuf[(*ulen)++] = buf[i]; - } - - return 1; -} - -static int -looks_latin1(const unsigned char *buf, size_t nbytes, my_unichar *ubuf, size_t *ulen) -{ - int i; - - *ulen = 0; - - for (i = 0; i < (int)nbytes; i++) { - int t = text_chars[buf[i]]; - - if (t != T && t != I) - return 0; - - ubuf[(*ulen)++] = buf[i]; - } - - return 1; -} - -static int -looks_extended(const unsigned char *buf, size_t nbytes, my_unichar *ubuf, - size_t *ulen) -{ - int i; - - *ulen = 0; - - for (i = 0; i < (int)nbytes; i++) { - int t = text_chars[buf[i]]; - - if (t != T && t != I && t != X) - return 0; - - ubuf[(*ulen)++] = buf[i]; - } - - return 1; -} - -static int -looks_utf8(const unsigned char *buf, size_t nbytes, my_unichar *ubuf, size_t *ulen) -{ - int i, n; - my_unichar c; - int gotone = 0; - - *ulen = 0; - - for (i = 0; i < (int)nbytes; i++) { - if ((buf[i] & 0x80) == 0) { /* 0xxxxxxx is plain ASCII */ - /* - * Even if the whole file is valid UTF-8 sequences, - * still reject it if it uses weird control characters. - */ - - if (text_chars[buf[i]] != T) - return 0; - - ubuf[(*ulen)++] = buf[i]; - } else if ((buf[i] & 0x40) == 0) { /* 10xxxxxx never 1st byte */ - return 0; - } else { /* 11xxxxxx begins UTF-8 */ - int following; - - if ((buf[i] & 0x20) == 0) { /* 110xxxxx */ - c = buf[i] & 0x1f; - following = 1; - } else if ((buf[i] & 0x10) == 0) { /* 1110xxxx */ - c = buf[i] & 0x0f; - following = 2; - } else if ((buf[i] & 0x08) == 0) { /* 11110xxx */ - c = buf[i] & 0x07; - following = 3; - } else if ((buf[i] & 0x04) == 0) { /* 111110xx */ - c = buf[i] & 0x03; - following = 4; - } else if ((buf[i] & 0x02) == 0) { /* 1111110x */ - c = buf[i] & 0x01; - following = 5; - } else - return 0; - - for (n = 0; n < following; n++) { - i++; - if (i >= (int)nbytes) - goto done; - - if ((buf[i] & 0x80) == 0 || (buf[i] & 0x40)) - return 0; - - c = (c << 6) + (buf[i] & 0x3f); - } - - ubuf[(*ulen)++] = c; - gotone = 1; - } - } -done: - return gotone; /* don't claim it's UTF-8 if it's all 7-bit */ -} - -static int -looks_unicode(const unsigned char *buf, size_t nbytes, my_unichar *ubuf, - size_t *ulen) -{ - int bigend; - int i; - - if (nbytes < 2) - return 0; - - if (buf[0] == 0xff && buf[1] == 0xfe) - bigend = 0; - else if (buf[0] == 0xfe && buf[1] == 0xff) - bigend = 1; - else - return 0; - - *ulen = 0; - - for (i = 2; i + 1 < (int)nbytes; i += 2) { - /* XXX fix to properly handle chars > 65536 */ - - if (bigend) - ubuf[(*ulen)++] = buf[i + 1] + 256 * buf[i]; - else - ubuf[(*ulen)++] = buf[i] + 256 * buf[i + 1]; - - if (ubuf[*ulen - 1] == 0xfffe) - return 0; - if (ubuf[*ulen - 1] < 128 && - text_chars[(size_t)ubuf[*ulen - 1]] != T) - return 0; - } - - return 1 + bigend; -} - -#undef F -#undef T -#undef I -#undef X - -/* - * This table maps each EBCDIC character to an (8-bit extended) ASCII - * character, as specified in the rationale for the dd(1) command in - * draft 11.2 (September, 1991) of the POSIX P1003.2 standard. - * - * Unfortunately it does not seem to correspond exactly to any of the - * five variants of EBCDIC documented in IBM's _Enterprise Systems - * Architecture/390: Principles of Operation_, SA22-7201-06, Seventh - * Edition, July, 1999, pp. I-1 - I-4. - * - * Fortunately, though, all versions of EBCDIC, including this one, agree - * on most of the printing characters that also appear in (7-bit) ASCII. - * Of these, only '|', '!', '~', '^', '[', and ']' are in question at all. - * - * Fortunately too, there is general agreement that codes 0x00 through - * 0x3F represent control characters, 0x41 a nonbreaking space, and the - * remainder printing characters. - * - * This is sufficient to allow us to identify EBCDIC text and to distinguish - * between old-style and internationalized examples of text. - */ - -static unsigned char ebcdic_to_ascii[] = { - 0, 1, 2, 3, 156, 9, 134, 127, 151, 141, 142, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 157, 133, 8, 135, 24, 25, 146, 143, 28, 29, 30, 31, -128, 129, 130, 131, 132, 10, 23, 27, 136, 137, 138, 139, 140, 5, 6, 7, -144, 145, 22, 147, 148, 149, 150, 4, 152, 153, 154, 155, 20, 21, 158, 26, -' ', 160, 161, 162, 163, 164, 165, 166, 167, 168, 213, '.', '<', '(', '+', '|', -'&', 169, 170, 171, 172, 173, 174, 175, 176, 177, '!', '$', '*', ')', ';', '~', -'-', '/', 178, 179, 180, 181, 182, 183, 184, 185, 203, ',', '%', '_', '>', '?', -186, 187, 188, 189, 190, 191, 192, 193, 194, '`', ':', '#', '@', '\'','=', '"', -195, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 196, 197, 198, 199, 200, 201, -202, 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', '^', 204, 205, 206, 207, 208, -209, 229, 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 210, 211, 212, '[', 214, 215, -216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, ']', 230, 231, -'{', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 232, 233, 234, 235, 236, 237, -'}', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 238, 239, 240, 241, 242, 243, -'\\',159, 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 244, 245, 246, 247, 248, 249, -'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 250, 251, 252, 253, 254, 255 -}; - -#ifdef notdef -/* - * The following EBCDIC-to-ASCII table may relate more closely to reality, - * or at least to modern reality. It comes from - * - * http://ftp.s390.ibm.com/products/oe/bpxqp9.html - * - * and maps the characters of EBCDIC code page 1047 (the code used for - * Unix-derived software on IBM's 390 systems) to the corresponding - * characters from ISO 8859-1. - * - * If this table is used instead of the above one, some of the special - * cases for the NEL character can be taken out of the code. - */ - -static unsigned char ebcdic_1047_to_8859[] = { -0x00,0x01,0x02,0x03,0x9C,0x09,0x86,0x7F,0x97,0x8D,0x8E,0x0B,0x0C,0x0D,0x0E,0x0F, -0x10,0x11,0x12,0x13,0x9D,0x0A,0x08,0x87,0x18,0x19,0x92,0x8F,0x1C,0x1D,0x1E,0x1F, -0x80,0x81,0x82,0x83,0x84,0x85,0x17,0x1B,0x88,0x89,0x8A,0x8B,0x8C,0x05,0x06,0x07, -0x90,0x91,0x16,0x93,0x94,0x95,0x96,0x04,0x98,0x99,0x9A,0x9B,0x14,0x15,0x9E,0x1A, -0x20,0xA0,0xE2,0xE4,0xE0,0xE1,0xE3,0xE5,0xE7,0xF1,0xA2,0x2E,0x3C,0x28,0x2B,0x7C, -0x26,0xE9,0xEA,0xEB,0xE8,0xED,0xEE,0xEF,0xEC,0xDF,0x21,0x24,0x2A,0x29,0x3B,0x5E, -0x2D,0x2F,0xC2,0xC4,0xC0,0xC1,0xC3,0xC5,0xC7,0xD1,0xA6,0x2C,0x25,0x5F,0x3E,0x3F, -0xF8,0xC9,0xCA,0xCB,0xC8,0xCD,0xCE,0xCF,0xCC,0x60,0x3A,0x23,0x40,0x27,0x3D,0x22, -0xD8,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0xAB,0xBB,0xF0,0xFD,0xFE,0xB1, -0xB0,0x6A,0x6B,0x6C,0x6D,0x6E,0x6F,0x70,0x71,0x72,0xAA,0xBA,0xE6,0xB8,0xC6,0xA4, -0xB5,0x7E,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7A,0xA1,0xBF,0xD0,0x5B,0xDE,0xAE, -0xAC,0xA3,0xA5,0xB7,0xA9,0xA7,0xB6,0xBC,0xBD,0xBE,0xDD,0xA8,0xAF,0x5D,0xB4,0xD7, -0x7B,0x41,0x42,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0xAD,0xF4,0xF6,0xF2,0xF3,0xF5, -0x7D,0x4A,0x4B,0x4C,0x4D,0x4E,0x4F,0x50,0x51,0x52,0xB9,0xFB,0xFC,0xF9,0xFA,0xFF, -0x5C,0xF7,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5A,0xB2,0xD4,0xD6,0xD2,0xD3,0xD5, -0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0xB3,0xDB,0xDC,0xD9,0xDA,0x9F -}; -#endif - -/* - * Copy buf[0 ... nbytes-1] into out[], translating EBCDIC to ASCII. - */ -static void -from_ebcdic(const unsigned char *buf, size_t nbytes, unsigned char *out) -{ - int i; - - for (i = 0; i < (int)nbytes; i++) { - out[i] = ebcdic_to_ascii[buf[i]]; - } -} - - // #pragma mark - @@ -889,10 +230,12 @@ identify_text(uint8* data, int32 bytesRead, BPositionIO* source, bytesRead += readLater; - // TODO: identify encoding as possible! - BMimeType type; - if (!file_ascmagic((const unsigned char*)data, bytesRead, &type, encoding)) + TextEncoding textEncoding((char*)data, (size_t)bytesRead); + encoding = textEncoding.GetName(); + if (strlen(encoding) == 0) { + /* No valid character encoding found! */ return B_NO_TRANSLATOR; + } float capability = TEXT_IN_CAPABILITY; if (bytesRead < 20) @@ -904,12 +247,8 @@ identify_text(uint8* data, int32 bytesRead, BPositionIO* source, outInfo->quality = TEXT_IN_QUALITY; outInfo->capability = capability; - char description[B_MIME_TYPE_LENGTH]; - if (type.GetLongDescription(description) == B_OK) - strlcpy(outInfo->name, description, sizeof(outInfo->name)); - else - strlcpy(outInfo->name, B_TRANSLATE("Plain text file"), - sizeof(outInfo->name)); + strlcpy(outInfo->name, B_TRANSLATE("Plain text file"), + sizeof(outInfo->name)); //strlcpy(outInfo->MIME, type.Type(), sizeof(outInfo->MIME)); strcpy(outInfo->MIME, "text/plain"); diff --git a/src/kits/locale/Jamfile b/src/kits/locale/Jamfile index c21abbaaf8..04f7c77a40 100644 --- a/src/kits/locale/Jamfile +++ b/src/kits/locale/Jamfile @@ -21,6 +21,7 @@ local sources = LocaleRoster.cpp LocaleRosterData.cpp MutableLocaleRoster.cpp + TextEncoding.cpp TimeZone.cpp # in progress diff --git a/src/kits/locale/TextEncoding.cpp b/src/kits/locale/TextEncoding.cpp new file mode 100644 index 0000000000..1cc25b39b7 --- /dev/null +++ b/src/kits/locale/TextEncoding.cpp @@ -0,0 +1,29 @@ +/* + * Copyright 2016, Haiku, inc. + * Distributed under terms of the MIT license. + */ + + +#include "TextEncoding.h" + +#include + + +TextEncoding::TextEncoding(const char* data, size_t length) +{ + UErrorCode error = U_ZERO_ERROR; + + UCharsetDetector* detector = ucsdet_open(&error); + ucsdet_setText(detector, data, length, &error); + const UCharsetMatch* encoding = ucsdet_detect(detector, &error); + + fName = ucsdet_getName(encoding, &error); + ucsdet_close(detector); +} + + +BString +TextEncoding::GetName() +{ + return fName; +}