i2p/haiku-beta6
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix GB18030 encoding support. And some cleanup ...

Browse Source 
* Fix GB18030 Chinese encoding support for two and four bytes long
  characters. This finally resolves issue described in #6227;
* Processing of multi-byte characters was slightly refactored too;
* Remove the multi-byte 94/96 graphsets designation support for
  Japanese encodings. That looks like MuTerm rudiment, it had incomplete
  implementation and looked like abandoned. On the other hand multi-byte
  designation must be implemented in the same way as designation for
  single-byte graphsets was done. Note that this multi-byte graphsets
  designation has nothing to do with the normal encoding support for
  usual data flow conversion - so you will be on the safe side when
  use terminal encoding menu switch.
  The removed feature is the ancient technique to achieve different charsets
  support on 8-bit serial lines by assigning (designating) predefined
  sets of characters to G0, G1, G2 and G3 and selecting them during
  program life-time into GL (x20-x07E) or GR (xA0-xFF) areas by using LS
  or SS functions.
  For example xterm has no support for designation multi-byte graphsets
  at all. Anyway if this feature is required and you can provide the
  test environment - please let me know and I will be glad to implement
  this feature in more easy and consistent way;
* Remove unreferenced gSmbcsTable and gScsTable parsing tables that
  looks like is not used anymore;
* Remove gCS96GroundTable and gMbcsTable parsing tables that were used
  by multi-byte 94/96 Japanese graphsets support and now obsoleted by
  removing mentioned feature;
* Remove some obsoleted #defines, like HW statusline support for
  example, from parse tables definition.
This commit is contained in:
Siarzhuk Zharski
2013-04-20 16:51:42 +02:00
parent 8c1b20b862
commit bf88d81ea6
3 changed files with 61 additions and 1559 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/apps/terminal/TermParse.cpp
+55 -103
View File
@@ -37,7 +37,6 @@
extern int gUTF8GroundTable[]; /* UTF8 Ground table */ extern int gUTF8GroundTable[]; /* UTF8 Ground table */
extern int gCS96GroundTable[]; /* CS96 Ground table */
extern int gISO8859GroundTable[]; /* ISO8859 & EUC Ground table */ extern int gISO8859GroundTable[]; /* ISO8859 & EUC Ground table */
extern int gWinCPGroundTable[]; /* Windows cp1252, cp1251, koi-8r */ extern int gWinCPGroundTable[]; /* Windows cp1252, cp1251, koi-8r */
extern int gSJISGroundTable[]; /* Shift-JIS Ground table */ extern int gSJISGroundTable[]; /* Shift-JIS Ground table */
@@ -49,7 +48,6 @@ extern int gScrTable[]; /* ESC # */
extern int gIgnoreTable[]; /* ignore table */ extern int gIgnoreTable[]; /* ignore table */
extern int gIesTable[]; /* ignore ESC table */ extern int gIesTable[]; /* ignore ESC table */
extern int gEscIgnoreTable[]; /* ESC ignore table */ extern int gEscIgnoreTable[]; /* ESC ignore table */
extern int gMbcsTable[]; /* ESC $ */
extern const char* gLineDrawGraphSet[]; /* may be used for G0, G1, G2, G3 */ extern const char* gLineDrawGraphSet[]; /* may be used for G0, G1, G2, G3 */
@@ -290,7 +288,6 @@ TermParse::DumpState(int *groundtable, int *parsestate, uchar c)
#define T(t) \ #define T(t) \
{ t, #t } { t, #t }
T(gUTF8GroundTable), T(gUTF8GroundTable),
T(gCS96GroundTable),
T(gISO8859GroundTable), T(gISO8859GroundTable),
T(gWinCPGroundTable), T(gWinCPGroundTable),
T(gSJISGroundTable), T(gSJISGroundTable),
@@ -301,7 +298,6 @@ TermParse::DumpState(int *groundtable, int *parsestate, uchar c)
T(gIgnoreTable), T(gIgnoreTable),
T(gIesTable), T(gIesTable),
T(gEscIgnoreTable), T(gEscIgnoreTable),
T(gMbcsTable),
{ NULL, NULL } { NULL, NULL }
}; };
int i; int i;
@@ -339,7 +335,6 @@ TermParse::_GuessGroundTable(int encoding)
case B_EUC_CONVERSION: case B_EUC_CONVERSION:
case B_EUC_KR_CONVERSION: case B_EUC_KR_CONVERSION:
case B_JIS_CONVERSION: case B_JIS_CONVERSION:
case B_GBK_CONVERSION:
case B_BIG5_CONVERSION: case B_BIG5_CONVERSION:
return gISO8859GroundTable; return gISO8859GroundTable;
@@ -348,6 +343,7 @@ TermParse::_GuessGroundTable(int encoding)
case B_MS_WINDOWS_CONVERSION: case B_MS_WINDOWS_CONVERSION:
case B_MAC_ROMAN_CONVERSION: case B_MAC_ROMAN_CONVERSION:
case B_MS_DOS_866_CONVERSION: case B_MS_DOS_866_CONVERSION:
case B_GBK_CONVERSION:
case B_MS_DOS_CONVERSION: case B_MS_DOS_CONVERSION:
return gWinCPGroundTable; return gWinCPGroundTable;
@@ -368,12 +364,9 @@ TermParse::EscParse()
{ {
int top; int top;
int bottom; int bottom;
// int cs96 = 0;
uchar curess = 0;
char cbuf[4] = { 0 }; char cbuf[4] = { 0 };
char dstbuf[4] = { 0 }; char dstbuf[4] = { 0 };
char *ptr;
int currentEncoding = -1; int currentEncoding = -1;
@@ -392,11 +385,6 @@ TermParse::EscParse()
int curGL = 0; int curGL = 0;
int curGR = 0; int curGR = 0;
int32 srcLen = sizeof(cbuf);
int32 dstLen = sizeof(dstbuf);
int32 dummyState = 0;
int width = 1;
BAutolock locker(fBuffer); BAutolock locker(fBuffer);
while (!fQuitting) { while (!fQuitting) {
@@ -413,6 +401,9 @@ TermParse::EscParse()
} }
//debug_printf("TermParse: char: '%c' (%d), parse state: %d\n", c, c, parsestate[c]); //debug_printf("TermParse: char: '%c' (%d), parse state: %d\n", c, c, parsestate[c]);
int32 srcLen = 0;
int32 dstLen = sizeof(dstbuf);
int32 dummyState = 0;
switch (parsestate[c]) { switch (parsestate[c]) {
case CASE_PRINT: case CASE_PRINT:
@@ -431,70 +422,48 @@ TermParse::EscParse()
break; break;
} }
case CASE_PRINT_GR: case CASE_PRINT_GR:
{
/* case iso8859 gr character, or euc */ /* case iso8859 gr character, or euc */
ptr = cbuf; switch (currentEncoding) {
if (currentEncoding == B_EUC_CONVERSION case B_EUC_CONVERSION:
|| currentEncoding == B_EUC_KR_CONVERSION case B_EUC_KR_CONVERSION:
|| currentEncoding == B_JIS_CONVERSION case B_JIS_CONVERSION:
|| currentEncoding == B_GBK_CONVERSION case B_BIG5_CONVERSION:
|| currentEncoding == B_BIG5_CONVERSION) { cbuf[srcLen++] = c;
switch (parsestate[curess]) { c = _NextParseChar();
case CASE_SS2: /* JIS X 0201 */ cbuf[srcLen++] = c;
width = 1; break;
*ptr++ = curess;
*ptr++ = c;
*ptr = 0;
curess = 0;
break;
case CASE_SS3: /* JIS X 0212 */ case B_GBK_CONVERSION:
width = 1; cbuf[srcLen++] = c;
*ptr++ = curess; do {
*ptr++ = c; // GBK-compatible codepoints are 2-bytes long
c = _NextParseChar(); c = _NextParseChar();
*ptr++ = c; cbuf[srcLen++] = c;
*ptr = 0;
curess = 0;
break;
default: /* JIS X 0208 */ // GB18030 extends GBK with 4-byte codepoints
width = 2; // using 2nd byte from range 0x30...0x39
*ptr++ = c; if (srcLen == 2 && (c < 0x30 || c > 0x39))
c = _NextParseChar(); break;
*ptr++ = c; } while (srcLen < 4);
*ptr = 0; break;
break;
} default: // ISO-8859-1...10 and MacRoman
} else { cbuf[srcLen++] = c;
/* ISO-8859-1...10 and MacRoman */ break;
*ptr++ = c;
*ptr = 0;
} }
srcLen = strlen(cbuf); if (srcLen > 0) {
dstLen = sizeof(dstbuf); int encoding = currentEncoding == B_JIS_CONVERSION
if (currentEncoding != B_JIS_CONVERSION) { ? B_EUC_CONVERSION : currentEncoding;
convert_to_utf8(currentEncoding, cbuf, &srcLen,
dstbuf, &dstLen, &dummyState, '?'); convert_to_utf8(encoding, cbuf, &srcLen,
} else {
convert_to_utf8(B_EUC_CONVERSION, cbuf, &srcLen,
dstbuf, &dstLen, &dummyState, '?'); dstbuf, &dstLen, &dummyState, '?');
fBuffer->InsertChar(UTF8Char(dstbuf, dstLen));
} }
fBuffer->InsertChar(UTF8Char(dstbuf, dstLen));
break;
case CASE_PRINT_CS96:
cbuf[0] = c | 0x80;
c = _NextParseChar();
cbuf[1] = c | 0x80;
cbuf[2] = 0;
srcLen = 2;
dstLen = sizeof(dstbuf);
convert_to_utf8(B_EUC_CONVERSION, cbuf, &srcLen,
dstbuf, &dstLen, &dummyState, '?');
fBuffer->InsertChar(UTF8Char(dstbuf, dstLen));
break; break;
}
case CASE_LF: case CASE_LF:
fBuffer->InsertLF(); fBuffer->InsertLF();
@@ -505,62 +474,47 @@ TermParse::EscParse()
break; break;
case CASE_SJIS_KANA: case CASE_SJIS_KANA:
cbuf[0] = c; cbuf[srcLen++] = c;
cbuf[1] = '\0';
srcLen = 1;
dstLen = sizeof(dstbuf);
convert_to_utf8(currentEncoding, cbuf, &srcLen, convert_to_utf8(currentEncoding, cbuf, &srcLen,
dstbuf, &dstLen, &dummyState, '?'); dstbuf, &dstLen, &dummyState, '?');
fBuffer->InsertChar(UTF8Char(dstbuf, dstLen)); fBuffer->InsertChar(UTF8Char(dstbuf, dstLen));
break; break;
case CASE_SJIS_INSTRING: case CASE_SJIS_INSTRING:
cbuf[0] = c; cbuf[srcLen++] = c;
c = _NextParseChar(); c = _NextParseChar();
cbuf[1] = c; cbuf[srcLen++] = c;
cbuf[2] = '\0';
srcLen = 2;
dstLen = sizeof(dstbuf);
convert_to_utf8(currentEncoding, cbuf, &srcLen, convert_to_utf8(currentEncoding, cbuf, &srcLen,
dstbuf, &dstLen, &dummyState, '?'); dstbuf, &dstLen, &dummyState, '?');
fBuffer->InsertChar(UTF8Char(dstbuf, dstLen)); fBuffer->InsertChar(UTF8Char(dstbuf, dstLen));
break; break;
case CASE_UTF8_2BYTE: case CASE_UTF8_2BYTE:
cbuf[0] = c; cbuf[srcLen++] = c;
c = _NextParseChar(); c = _NextParseChar();
if (groundtable[c] != CASE_UTF8_INSTRING) if (groundtable[c] != CASE_UTF8_INSTRING)
break; break;
cbuf[1] = c; cbuf[srcLen++] = c;
cbuf[2] = '\0';
fBuffer->InsertChar(UTF8Char(cbuf, 2)); fBuffer->InsertChar(UTF8Char(cbuf, srcLen));
break; break;
case CASE_UTF8_3BYTE: case CASE_UTF8_3BYTE:
cbuf[0] = c; cbuf[srcLen++] = c;
c = _NextParseChar();
if (groundtable[c] != CASE_UTF8_INSTRING)
break;
cbuf[1] = c;
c = _NextParseChar(); do {
if (groundtable[c] != CASE_UTF8_INSTRING) c = _NextParseChar();
break; if (groundtable[c] != CASE_UTF8_INSTRING) {
cbuf[2] = c; srcLen = 0;
cbuf[3] = '\0'; break;
fBuffer->InsertChar(UTF8Char(cbuf, 3)); }
break; cbuf[srcLen++] = c;
case CASE_MBCS: } while (srcLen != 3);
/* ESC $ */
parsestate = gMbcsTable;
break;
case CASE_GSETS: if (srcLen > 0)
/* ESC $ ? */ fBuffer->InsertChar(UTF8Char(cbuf, srcLen));
parsestate = gCS96GroundTable;
// cs96 = 1;
break; break;
case CASE_SCS_STATE: case CASE_SCS_STATE:
@@ -1073,13 +1027,11 @@ TermParse::EscParse()
case CASE_SS2: case CASE_SS2:
/* SS2 */ /* SS2 */
curess = c;
parsestate = groundtable; parsestate = groundtable;
break; break;
case CASE_SS3: case CASE_SS3:
/* SS3 */ /* SS3 */
curess = c;
parsestate = groundtable; parsestate = groundtable;
break; break;
src/apps/terminal/VTPrsTbl.c
+6 -1444
View File
File diff suppressed because it is too large Load Diff
src/apps/terminal/VTparse.h
-12
View File
@@ -25,10 +25,6 @@
#define CASE_LS1 12 #define CASE_LS1 12
#define CASE_SP 13 #define CASE_SP 13
#define CASE_SCR_STATE 14 #define CASE_SCR_STATE 14
#define CASE_SCS0_STATE 15
#define CASE_SCS1_STATE 16
#define CASE_SCS2_STATE 17
#define CASE_SCS3_STATE 18
#define CASE_ESC_IGNORE 19 #define CASE_ESC_IGNORE 19
#define CASE_ESC_DIGIT 20 #define CASE_ESC_DIGIT 20
#define CASE_ESC_SEMI 21 #define CASE_ESC_SEMI 21
@@ -56,7 +52,6 @@
#define CASE_DECSET 43 #define CASE_DECSET 43
#define CASE_DECRST 44 #define CASE_DECRST 44
#define CASE_DECALN 45 #define CASE_DECALN 45
#define CASE_GSETS 46
#define CASE_DECSC 47 #define CASE_DECSC 47
#define CASE_DECRC 48 #define CASE_DECRC 48
#define CASE_DECKPAM 49 #define CASE_DECKPAM 49
@@ -83,12 +78,6 @@
#define CASE_HP_MEM_LOCK 70 #define CASE_HP_MEM_LOCK 70
#define CASE_HP_MEM_UNLOCK 71 #define CASE_HP_MEM_UNLOCK 71
#define CASE_HP_BUGGY_LL 72 #define CASE_HP_BUGGY_LL 72
#define CASE_TO_STATUS 73
#define CASE_FROM_STATUS 74
#define CASE_SHOW_STATUS 75
#define CASE_HIDE_STATUS 76
#define CASE_ERASE_STATUS 77
#define CASE_MBCS 78
#define CASE_SCS_STATE 79 #define CASE_SCS_STATE 79
#define CASE_UTF8_2BYTE 80 #define CASE_UTF8_2BYTE 80
#define CASE_UTF8_3BYTE 81 #define CASE_UTF8_3BYTE 81
@@ -96,7 +85,6 @@
#define CASE_SJIS_INSTRING 83 #define CASE_SJIS_INSTRING 83
#define CASE_SJIS_KANA 84 #define CASE_SJIS_KANA 84
#define CASE_PRINT_GR 85 #define CASE_PRINT_GR 85
#define CASE_PRINT_CS96 86
// additions, maybe reorder/reuse older ones ? // additions, maybe reorder/reuse older ones ?
#define CASE_VPA 87 #define CASE_VPA 87
#define CASE_HPA 88 #define CASE_HPA 88