Author: atsushi
Date: 2006-01-20 05:14:26 -0500 (Fri, 20 Jan 2006)
New Revision: 55827
Modified:
trunk/mcs/class/I18N/CJK/ChangeLog
trunk/mcs/class/I18N/CJK/GB18030Encoding.cs
trunk/mcs/class/I18N/CJK/GB18030Source.cs
trunk/mcs/class/I18N/CJK/Test/ChangeLog
trunk/mcs/class/I18N/CJK/Test/I18N.CJK.Test.cs
Log:
2006-01-20 Atsushi Enomoto <[EMAIL PROTECTED]>
Fixed bug #77307.
* GB18030Encoding.cs : GetEncoder() and GetDecoder() should be
overriden. in the Encoder, refresh parameter should handle
incomplete field at its end of processing. Also it should
differentiate GetByteCount() and GetChars() for its internal
state.
* GB18030Source.cs : handle end index of GB18030Map as inclusive,
not exclusive. Reject out-of-range GBX conversion.
* I18N.CJK.Test.cs : Added test for #77307. Unify those "get bytes
for all of chars" tests.
Modified: trunk/mcs/class/I18N/CJK/ChangeLog
===================================================================
--- trunk/mcs/class/I18N/CJK/ChangeLog 2006-01-20 08:58:54 UTC (rev 55826)
+++ trunk/mcs/class/I18N/CJK/ChangeLog 2006-01-20 10:14:26 UTC (rev 55827)
@@ -1,9 +1,20 @@
2006-01-20 Atsushi Enomoto <[EMAIL PROTECTED]>
- * GB18030.cs : GetChars() ditto. Fixed bug #77306.
+ Fixed bug #77307.
+ * GB18030Encoding.cs : GetEncoder() and GetDecoder() should be
+ overriden. in the Encoder, refresh parameter should handle
+ incomplete field at its end of processing. Also it should
+ differentiate GetByteCount() and GetChars() for its internal
+ state.
+ * GB18030Source.cs : handle end index of GB18030Map as inclusive,
+ not exclusive. Reject out-of-range GBX conversion.
2006-01-20 Atsushi Enomoto <[EMAIL PROTECTED]>
+ * GB18030Encoding.cs : GetChars() ditto. Fixed bug #77306.
+
+2006-01-20 Atsushi Enomoto <[EMAIL PROTECTED]>
+
* CP949.cs : GetChars() exactly the same fix as CP950.
Fixed bug #77298.
Modified: trunk/mcs/class/I18N/CJK/GB18030Encoding.cs
===================================================================
--- trunk/mcs/class/I18N/CJK/GB18030Encoding.cs 2006-01-20 08:58:54 UTC (rev
55826)
+++ trunk/mcs/class/I18N/CJK/GB18030Encoding.cs 2006-01-20 10:14:26 UTC (rev
55827)
@@ -67,6 +67,16 @@
{
return new GB18030Decoder ().GetChars (bytes, byteIdx,
srclen, chars, charIdx);
}
+
+ public override Encoder GetEncoder ()
+ {
+ return new GB18030Encoder (this);
+ }
+
+ public override Decoder GetDecoder ()
+ {
+ return new GB18030Decoder ();
+ }
}
class GB18030Decoder : Decoder
@@ -253,13 +263,11 @@
{
}
- char incomplete;
+ char incomplete_byte_count;
+ char incomplete_bytes;
public override int GetByteCount (char [] chars, int start, int
len, bool refresh)
{
- if (refresh)
- incomplete = char.MinValue;
-
if (chars == null)
throw new ArgumentNullException ("chars");
if (start < 0 || start > chars.Length)
@@ -278,10 +286,13 @@
continue;
} else if (Char.IsSurrogate (ch)) {
// Surrogate
- if (start + 1 == end)
- break; // incomplete
- ret += 4;
- start += 2;
+ if (start + 1 == end) {
+ incomplete_byte_count = ch;
+ start++;
+ } else {
+ ret += 4;
+ start += 2;
+ }
continue;
}
@@ -302,13 +313,23 @@
}
// non-GB2312
- ret += 4;
+ long value = GB18030Source.FromUCS (ch);
+ if (value < 0)
+ ret++; // invalid(?)
+ else
+ ret += 4;
start++;
}
+
+ if (refresh) {
+ if (incomplete_byte_count != char.MinValue)
+ ret++;
+ incomplete_byte_count = char.MinValue;
+ }
return ret;
}
- public unsafe override int GetBytesImpl (char* chars, int
charCount, byte* bytes, int byteCount, bool flush)
+ public unsafe override int GetBytesImpl (char* chars, int
charCount, byte* bytes, int byteCount, bool refresh)
{
int charIndex = 0;
int byteIndex = 0;
@@ -318,13 +339,13 @@
int charEnd = charIndex + charCount;
int byteStart = byteIndex;
- char ch = incomplete;
+ char ch = incomplete_bytes;
while (charIndex < charEnd) {
- if (incomplete == char.MinValue)
+ if (incomplete_bytes == char.MinValue)
ch = chars [charIndex++];
else
- incomplete = char.MinValue;
+ incomplete_bytes = char.MinValue;
if (ch < 0x80) {
// ASCII
@@ -333,7 +354,7 @@
} else if (Char.IsSurrogate (ch)) {
// Surrogate
if (charIndex == charEnd) {
- incomplete = ch;
+ incomplete_bytes = ch;
break; // incomplete
}
char ch2 = chars [charIndex++];
@@ -370,10 +391,21 @@
}
long value = GB18030Source.FromUCS (ch);
- // non-GB2312
- GB18030Source.Unlinear (bytes + byteIndex,
value);
- byteIndex += 4;
+ if (value < 0)
+ bytes [byteIndex++] = 0x3F; //
invalid(?)
+ else {
+ // non-GB2312
+ GB18030Source.Unlinear (bytes +
byteIndex, value);
+ byteIndex += 4;
+ }
}
+
+ if (refresh) {
+ if (incomplete_bytes != char.MinValue)
+ bytes [byteIndex++] = 0x3F; //
incomplete
+ incomplete_bytes = char.MinValue;
+ }
+
return byteIndex - byteStart;
}
}
Modified: trunk/mcs/class/I18N/CJK/GB18030Source.cs
===================================================================
--- trunk/mcs/class/I18N/CJK/GB18030Source.cs 2006-01-20 08:58:54 UTC (rev
55826)
+++ trunk/mcs/class/I18N/CJK/GB18030Source.cs 2006-01-20 10:14:26 UTC (rev
55827)
@@ -42,6 +42,7 @@
static readonly byte *gbx2uni;
static readonly byte *uni2gbx;
+ static readonly int gbx2uniSize, uni2gbxSize;
static GB18030Source ()
{
@@ -55,11 +56,15 @@
new object [] {"gb18030.table", size, mod});
if (ret != IntPtr.Zero) {
gbx2uni = (byte*) ((void*) ret);
- int gbx2uniSize =
+ gbx2uniSize =
(gbx2uni [0] << 24) + (gbx2uni [1] <<
16) +
(gbx2uni [2] << 8) + (gbx2uni [3]);
gbx2uni += 4;
uni2gbx = gbx2uni + gbx2uniSize;
+ uni2gbxSize =
+ (uni2gbx [0] << 24) + (uni2gbx [1] <<
16) +
+ (uni2gbx [2] << 8) + (uni2gbx [3]);
+ uni2gbx += 4;
}
}
@@ -151,7 +156,7 @@
if (linear < m.GStart)
return ToUcsRaw ((int) (linear
- startIgnore + rawOffset));
- if (linear < m.GEnd)
+ if (linear <= m.GEnd)
return linear - gbxBase - m.GStart
+ m.UStart;
if (m.GStart != 0) {
@@ -177,7 +182,7 @@
if (cp < m.UStart)
return ToGbxRaw ((int) (cp
- startIgnore + rawOffset));
- if (cp < m.UEnd)
+ if (cp <= m.UEnd)
return cp - m.UStart + m.GStart;
if (m.GStart != 0) {
rawOffset += m.UStart - startIgnore;
@@ -205,6 +210,8 @@
static long ToGbxRaw (int idx)
{
+ if (idx < 0 || idx * 2 + 1 >= uni2gbxSize)
+ return -1;
return gbxBase + uni2gbx [idx * 2] * 0x100 + uni2gbx
[idx * 2 + 1];
}
Modified: trunk/mcs/class/I18N/CJK/Test/ChangeLog
===================================================================
--- trunk/mcs/class/I18N/CJK/Test/ChangeLog 2006-01-20 08:58:54 UTC (rev
55826)
+++ trunk/mcs/class/I18N/CJK/Test/ChangeLog 2006-01-20 10:14:26 UTC (rev
55827)
@@ -1,5 +1,10 @@
2006-01-20 Atsushi Enomoto <[EMAIL PROTECTED]>
+ * I18N.CJK.Test.cs : Added test for #77307. Unify those "get bytes
+ for all of chars" tests.
+
+2006-01-20 Atsushi Enomoto <[EMAIL PROTECTED]>
+
* I18N.CJK.Test.cs : Added test for #77306.
2006-01-20 Atsushi Enomoto <[EMAIL PROTECTED]>
Modified: trunk/mcs/class/I18N/CJK/Test/I18N.CJK.Test.cs
===================================================================
--- trunk/mcs/class/I18N/CJK/Test/I18N.CJK.Test.cs 2006-01-20 08:58:54 UTC
(rev 55826)
+++ trunk/mcs/class/I18N/CJK/Test/I18N.CJK.Test.cs 2006-01-20 10:14:26 UTC
(rev 55827)
@@ -195,17 +195,26 @@
[Test]
public void Bug77723 ()
{
- Encoding e = Encoding.GetEncoding (51932);
- for (int i = 0; i < 0x10000; i++)
- e.GetBytes (new char [] { (char)i });
+ GetBytesAllSingleChars (51932);
}
[Test]
- public void Bug77224 ()
+ public void Bug77724 ()
{
- Encoding e = Encoding.GetEncoding (932);
+ GetBytesAllSingleChars (932);
+ }
+
+ [Test]
+ public void Bug77307 ()
+ {
+ GetBytesAllSingleChars (54936);
+ }
+
+ void GetBytesAllSingleChars (int enc)
+ {
+ Encoding e = Encoding.GetEncoding (enc);
for (int i = 0; i < 0x10000; i++)
- e.GetBytes (new char [] {(char) i});
+ e.GetBytes (new char [] { (char)i });
}
void GetCharsAllBytePairs (int enc)
_______________________________________________
Mono-patches maillist - [email protected]
http://lists.ximian.com/mailman/listinfo/mono-patches