i2p/haiku-beta6
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix GB18030 encoding support. And some cleanup ...

Browse Source 
* Fix GB18030 Chinese encoding support for two and four bytes long
  characters. This finally resolves issue described in #6227;
* Processing of multi-byte characters was slightly refactored too;
* Remove the multi-byte 94/96 graphsets designation support for
  Japanese encodings. That looks like MuTerm rudiment, it had incomplete
  implementation and looked like abandoned. On the other hand multi-byte
  designation must be implemented in the same way as designation for
  single-byte graphsets was done. Note that this multi-byte graphsets
  designation has nothing to do with the normal encoding support for
  usual data flow conversion - so you will be on the safe side when
  use terminal encoding menu switch.
  The removed feature is the ancient technique to achieve different charsets
  support on 8-bit serial lines by assigning (designating) predefined
  sets of characters to G0, G1, G2 and G3 and selecting them during
  program life-time into GL (x20-x07E) or GR (xA0-xFF) areas by using LS
  or SS functions.
  For example xterm has no support for designation multi-byte graphsets
  at all. Anyway if this feature is required and you can provide the
  test environment - please let me know and I will be glad to implement
  this feature in more easy and consistent way;
* Remove unreferenced gSmbcsTable and gScsTable parsing tables that
  looks like is not used anymore;
* Remove gCS96GroundTable and gMbcsTable parsing tables that were used
  by multi-byte 94/96 Japanese graphsets support and now obsoleted by
  removing mentioned feature;
* Remove some obsoleted #defines, like HW statusline support for
  example, from parse tables definition.
This commit is contained in:
Siarzhuk Zharski
2013-04-20 16:51:42 +02:00
parent 8c1b20b862
commit bf88d81ea6
3 changed files with 61 additions and 1559 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/apps/terminal/TermParse.cpp
+55 -103
View File
@@ -37,7 +37,6 @@
extern int gUTF8GroundTable[]; /* UTF8 Ground table */
extern int gCS96GroundTable[]; /* CS96 Ground table */
extern int gISO8859GroundTable[]; /* ISO8859 & EUC Ground table */
extern int gWinCPGroundTable[]; /* Windows cp1252, cp1251, koi-8r */
extern int gSJISGroundTable[]; /* Shift-JIS Ground table */
@@ -49,7 +48,6 @@ extern int gScrTable[]; /* ESC # */
extern int gIgnoreTable[]; /* ignore table */
extern int gIesTable[]; /* ignore ESC table */
extern int gEscIgnoreTable[]; /* ESC ignore table */
extern int gMbcsTable[]; /* ESC $ */
extern const char* gLineDrawGraphSet[]; /* may be used for G0, G1, G2, G3 */
@@ -290,7 +288,6 @@ TermParse::DumpState(int *groundtable, int *parsestate, uchar c)
#define T(t) \
{ t, #t }
T(gUTF8GroundTable),
T(gCS96GroundTable),
T(gISO8859GroundTable),
T(gWinCPGroundTable),
T(gSJISGroundTable),
@@ -301,7 +298,6 @@ TermParse::DumpState(int *groundtable, int *parsestate, uchar c)
T(gIgnoreTable),
T(gIesTable),
T(gEscIgnoreTable),
T(gMbcsTable),
{ NULL, NULL }
};
int i;
@@ -339,7 +335,6 @@ TermParse::_GuessGroundTable(int encoding)
case B_EUC_CONVERSION:
case B_EUC_KR_CONVERSION:
case B_JIS_CONVERSION:
case B_GBK_CONVERSION:
case B_BIG5_CONVERSION:
return gISO8859GroundTable;
@@ -348,6 +343,7 @@ TermParse::_GuessGroundTable(int encoding)
case B_MS_WINDOWS_CONVERSION:
case B_MAC_ROMAN_CONVERSION:
case B_MS_DOS_866_CONVERSION:
case B_GBK_CONVERSION:
case B_MS_DOS_CONVERSION:
return gWinCPGroundTable;
@@ -368,12 +364,9 @@ TermParse::EscParse()
{
int top;
int bottom;
// int cs96 = 0;
uchar curess = 0;
char cbuf[4] = { 0 };
char dstbuf[4] = { 0 };
char *ptr;
int currentEncoding = -1;
@@ -392,11 +385,6 @@ TermParse::EscParse()
int curGL = 0;
int curGR = 0;
int32 srcLen = sizeof(cbuf);
int32 dstLen = sizeof(dstbuf);
int32 dummyState = 0;
int width = 1;
BAutolock locker(fBuffer);
while (!fQuitting) {
@@ -413,6 +401,9 @@ TermParse::EscParse()
}
//debug_printf("TermParse: char: '%c' (%d), parse state: %d\n", c, c, parsestate[c]);
int32 srcLen = 0;
int32 dstLen = sizeof(dstbuf);
int32 dummyState = 0;
switch (parsestate[c]) {
case CASE_PRINT:
@@ -431,70 +422,48 @@ TermParse::EscParse()
break;
}
case CASE_PRINT_GR:
{
/* case iso8859 gr character, or euc */
ptr = cbuf;
if (currentEncoding == B_EUC_CONVERSION
|| currentEncoding == B_EUC_KR_CONVERSION
|| currentEncoding == B_JIS_CONVERSION
|| currentEncoding == B_GBK_CONVERSION
|| currentEncoding == B_BIG5_CONVERSION) {
switch (parsestate[curess]) {
case CASE_SS2: /* JIS X 0201 */
width = 1;
*ptr++ = curess;
*ptr++ = c;
*ptr = 0;
curess = 0;
break;
switch (currentEncoding) {
case B_EUC_CONVERSION:
case B_EUC_KR_CONVERSION:
case B_JIS_CONVERSION:
case B_BIG5_CONVERSION:
cbuf[srcLen++] = c;
c = _NextParseChar();
cbuf[srcLen++] = c;
break;
case CASE_SS3: /* JIS X 0212 */
width = 1;
*ptr++ = curess;
*ptr++ = c;
case B_GBK_CONVERSION:
cbuf[srcLen++] = c;
do {
// GBK-compatible codepoints are 2-bytes long
c = _NextParseChar();
*ptr++ = c;
*ptr = 0;
curess = 0;
break;
cbuf[srcLen++] = c;
default: /* JIS X 0208 */
width = 2;
*ptr++ = c;
c = _NextParseChar();
*ptr++ = c;
*ptr = 0;
break;
}
} else {
/* ISO-8859-1...10 and MacRoman */
*ptr++ = c;
*ptr = 0;
// GB18030 extends GBK with 4-byte codepoints
// using 2nd byte from range 0x30...0x39
if (srcLen == 2 && (c < 0x30 || c > 0x39))
break;
} while (srcLen < 4);
break;
default: // ISO-8859-1...10 and MacRoman
cbuf[srcLen++] = c;
break;
}
srcLen = strlen(cbuf);
dstLen = sizeof(dstbuf);
if (currentEncoding != B_JIS_CONVERSION) {
convert_to_utf8(currentEncoding, cbuf, &srcLen,
dstbuf, &dstLen, &dummyState, '?');
} else {
convert_to_utf8(B_EUC_CONVERSION, cbuf, &srcLen,
if (srcLen > 0) {
int encoding = currentEncoding == B_JIS_CONVERSION
? B_EUC_CONVERSION : currentEncoding;
convert_to_utf8(encoding, cbuf, &srcLen,
dstbuf, &dstLen, &dummyState, '?');
fBuffer->InsertChar(UTF8Char(dstbuf, dstLen));
}
fBuffer->InsertChar(UTF8Char(dstbuf, dstLen));
break;
case CASE_PRINT_CS96:
cbuf[0] = c | 0x80;
c = _NextParseChar();
cbuf[1] = c | 0x80;
cbuf[2] = 0;
srcLen = 2;
dstLen = sizeof(dstbuf);
convert_to_utf8(B_EUC_CONVERSION, cbuf, &srcLen,
dstbuf, &dstLen, &dummyState, '?');
fBuffer->InsertChar(UTF8Char(dstbuf, dstLen));
break;
}
case CASE_LF:
fBuffer->InsertLF();
@@ -505,62 +474,47 @@ TermParse::EscParse()
break;
case CASE_SJIS_KANA:
cbuf[0] = c;
cbuf[1] = '\0';
srcLen = 1;
dstLen = sizeof(dstbuf);
cbuf[srcLen++] = c;
convert_to_utf8(currentEncoding, cbuf, &srcLen,
dstbuf, &dstLen, &dummyState, '?');
fBuffer->InsertChar(UTF8Char(dstbuf, dstLen));
break;
case CASE_SJIS_INSTRING:
cbuf[0] = c;
cbuf[srcLen++] = c;
c = _NextParseChar();
cbuf[1] = c;
cbuf[2] = '\0';
srcLen = 2;
dstLen = sizeof(dstbuf);
cbuf[srcLen++] = c;
convert_to_utf8(currentEncoding, cbuf, &srcLen,
dstbuf, &dstLen, &dummyState, '?');
fBuffer->InsertChar(UTF8Char(dstbuf, dstLen));
break;
case CASE_UTF8_2BYTE:
cbuf[0] = c;
cbuf[srcLen++] = c;
c = _NextParseChar();
if (groundtable[c] != CASE_UTF8_INSTRING)
break;
cbuf[1] = c;
cbuf[2] = '\0';
cbuf[srcLen++] = c;
fBuffer->InsertChar(UTF8Char(cbuf, 2));
fBuffer->InsertChar(UTF8Char(cbuf, srcLen));
break;
case CASE_UTF8_3BYTE:
cbuf[0] = c;
c = _NextParseChar();
if (groundtable[c] != CASE_UTF8_INSTRING)
break;
cbuf[1] = c;
cbuf[srcLen++] = c;
c = _NextParseChar();
if (groundtable[c] != CASE_UTF8_INSTRING)
break;
cbuf[2] = c;
cbuf[3] = '\0';
fBuffer->InsertChar(UTF8Char(cbuf, 3));
break;
do {
c = _NextParseChar();
if (groundtable[c] != CASE_UTF8_INSTRING) {
srcLen = 0;
break;
}
cbuf[srcLen++] = c;
case CASE_MBCS:
/* ESC $ */
parsestate = gMbcsTable;
break;
} while (srcLen != 3);
case CASE_GSETS:
/* ESC $ ? */
parsestate = gCS96GroundTable;
// cs96 = 1;
if (srcLen > 0)
fBuffer->InsertChar(UTF8Char(cbuf, srcLen));
break;
case CASE_SCS_STATE:
@@ -1073,13 +1027,11 @@ TermParse::EscParse()
case CASE_SS2:
/* SS2 */
curess = c;
parsestate = groundtable;
break;
case CASE_SS3:
/* SS3 */
curess = c;
parsestate = groundtable;
break;
src/apps/terminal/VTPrsTbl.c
+6 -1444
View File
File diff suppressed because it is too large Load Diff
src/apps/terminal/VTparse.h
-12
View File
@@ -25,10 +25,6 @@
#define CASE_LS1 12
#define CASE_SP 13
#define CASE_SCR_STATE 14
#define CASE_SCS0_STATE 15
#define CASE_SCS1_STATE 16
#define CASE_SCS2_STATE 17
#define CASE_SCS3_STATE 18
#define CASE_ESC_IGNORE 19
#define CASE_ESC_DIGIT 20
#define CASE_ESC_SEMI 21
@@ -56,7 +52,6 @@
#define CASE_DECSET 43
#define CASE_DECRST 44
#define CASE_DECALN 45
#define CASE_GSETS 46
#define CASE_DECSC 47
#define CASE_DECRC 48
#define CASE_DECKPAM 49
@@ -83,12 +78,6 @@
#define CASE_HP_MEM_LOCK 70
#define CASE_HP_MEM_UNLOCK 71
#define CASE_HP_BUGGY_LL 72
#define CASE_TO_STATUS 73
#define CASE_FROM_STATUS 74
#define CASE_SHOW_STATUS 75
#define CASE_HIDE_STATUS 76
#define CASE_ERASE_STATUS 77
#define CASE_MBCS 78
#define CASE_SCS_STATE 79
#define CASE_UTF8_2BYTE 80
#define CASE_UTF8_3BYTE 81
@@ -96,7 +85,6 @@
#define CASE_SJIS_INSTRING 83
#define CASE_SJIS_KANA 84
#define CASE_PRINT_GR 85
#define CASE_PRINT_CS96 86
// additions, maybe reorder/reuse older ones ?
#define CASE_VPA 87
#define CASE_HPA 88