Author: atsushi
Date: 2005-07-15 01:37:18 -0400 (Fri, 15 Jul 2005)
New Revision: 47327
Modified:
trunk/mcs/class/corlib/Mono.Globalization.Unicode/ChangeLog
trunk/mcs/class/corlib/Mono.Globalization.Unicode/Collation-notes.txt
trunk/mcs/class/corlib/Mono.Globalization.Unicode/SimpleCollator.cs
trunk/mcs/class/corlib/Mono.Globalization.Unicode/TestDriver.cs
trunk/mcs/class/corlib/Test/System.Globalization/ChangeLog
trunk/mcs/class/corlib/Test/System.Globalization/CompareInfoTest.cs
Log:
2005-07-15 Atsushi Enomoto <[EMAIL PROTECTED]>
* SimpleCollator.cs : rewrote LastIndexOf() to handle source extenders.
Some refactoring on IndexOf() code. Removed unused Matches().
* Collation-notes.txt : some methods needed to be reimplemented, so
rewrote the description.
* CompareInfoTest.cs : Uncommented tests that got working. Added two
cornercase notworking LastIndexOf() tests.
Modified: trunk/mcs/class/corlib/Mono.Globalization.Unicode/ChangeLog
===================================================================
--- trunk/mcs/class/corlib/Mono.Globalization.Unicode/ChangeLog 2005-07-15
04:55:23 UTC (rev 47326)
+++ trunk/mcs/class/corlib/Mono.Globalization.Unicode/ChangeLog 2005-07-15
05:37:18 UTC (rev 47327)
@@ -1,3 +1,10 @@
+2005-07-15 Atsushi Enomoto <[EMAIL PROTECTED]>
+
+ * SimpleCollator.cs : rewrote LastIndexOf() to handle source extenders.
+ Some refactoring on IndexOf() code. Removed unused Matches().
+ * Collation-notes.txt : some methods needed to be reimplemented, so
+ rewrote the description.
+
2005-07-14 Atsushi Enomoto <[EMAIL PROTECTED]>
* SimpleCollator.cs : rewrote IsSuffix() to use CompareInternal().
Modified: trunk/mcs/class/corlib/Mono.Globalization.Unicode/Collation-notes.txt
===================================================================
--- trunk/mcs/class/corlib/Mono.Globalization.Unicode/Collation-notes.txt
2005-07-15 04:55:23 UTC (rev 47326)
+++ trunk/mcs/class/corlib/Mono.Globalization.Unicode/Collation-notes.txt
2005-07-15 05:37:18 UTC (rev 47327)
@@ -36,13 +36,13 @@
Find first difference and compare it.
"Larger/smaller" matters (beyond "different").
IsPrefix()
- For each character in the target, examine if it matches from
- the head of the searchee and immediately return false if it
- didn't match.
+ It calls CompareInternal() which also answers if the target
+ is fully consumed, so it just returns true if it says that
+ the target is fully consumed.
IsSuffix()
- For each character in the target, examine if it matches
- form the tail of the searchee and immediately return false if
- it didn't match. It uses reversal element query.
+ It tries CompareInternal() to compare source and target at
+ the end, where source varies from minimum tail to the
+ original args.
IndexOf(), LastIndexOf()
For character search, it finds the matching character element
to the end (or start) of the string to find.
Modified: trunk/mcs/class/corlib/Mono.Globalization.Unicode/SimpleCollator.cs
===================================================================
--- trunk/mcs/class/corlib/Mono.Globalization.Unicode/SimpleCollator.cs
2005-07-15 04:55:23 UTC (rev 47326)
+++ trunk/mcs/class/corlib/Mono.Globalization.Unicode/SimpleCollator.cs
2005-07-15 05:37:18 UTC (rev 47327)
@@ -1251,13 +1251,6 @@
#region IndexOf() / LastIndexOf()
- // IndexOf (string, string, CompareOptions)
- // IndexOf (string, string, int, int, CompareOptions)
- // IndexOf (string, char, int, int, CompareOptions)
- // IndexOfPrimitiveChar (string, int, int, char)
- // IndexOfSortKey (string, int, int, byte[], char, int, bool)
- // IndexOf (string, string, int, int)
-
public int IndexOf (string s, string target, CompareOptions opt)
{
return IndexOf (s, target, 0, s.Length, opt);
@@ -1287,23 +1280,20 @@
(opt &
CompareOptions.StringSort) != 0);
else
return IndexOfSortKey (s, start,
length, ct.SortKey, char.MinValue, -1, true);
+ } else {
+ int ti = FilterOptions ((int) target);
+ charSortKeyIndexTarget [0] = Category (ti);
+ charSortKeyIndexTarget [1] = Level1 (ti);
+ if (!ignoreNonSpace)
+ charSortKeyIndexTarget [2] =
+ Level2 (ti, ExtenderType.None);
+ charSortKeyIndexTarget [3] = Uni.Level3 (ti);
+ return IndexOfSortKey (s, start, length,
+ charSortKeyIndexTarget, target, ti,
+ !Uni.HasSpecialWeight ((char) ti));
}
- else
- return IndexOfPrimitiveChar (s, start, length,
target);
}
- // Searches target char w/o checking contractions
- int IndexOfPrimitiveChar (string s, int start, int length, char
target)
- {
- int ti = FilterOptions ((int) target);
- charSortKeyIndexTarget [0] = Category (ti);
- charSortKeyIndexTarget [1] = Level1 (ti);
- if (!ignoreNonSpace)
- charSortKeyIndexTarget [2] = Level2 (ti,
ExtenderType.None);
- charSortKeyIndexTarget [3] = Uni.Level3 (ti);
- return IndexOfSortKey (s, start, length,
charSortKeyIndexTarget, target, ti, !Uni.HasSpecialWeight ((char) ti));
- }
-
// Searches target byte[] keydata
int IndexOfSortKey (string s, int start, int length, byte []
sortkey, char target, int ti, bool noLv4)
{
@@ -1412,33 +1402,27 @@
if (ct.Replacement != null)
return LastIndexOf (s, ct.Replacement,
start, length, (opt & CompareOptions.StringSort) != 0);
else
- return LastIndexOfSortKey (s, start,
length, ct.SortKey, char.MinValue, -1, true);
+ return LastIndexOfSortKey (s, start,
start, length, ct.SortKey, char.MinValue, -1, true);
}
- else
- return LastIndexOfPrimitiveChar (s, start,
length, target);
+ else {
+ int ti = FilterOptions ((int) target);
+ charSortKeyIndexTarget [0] = Category (ti);
+ charSortKeyIndexTarget [1] = Level1 (ti);
+ if (!ignoreNonSpace)
+ charSortKeyIndexTarget [2] = Level2
(ti, ExtenderType.None);
+ charSortKeyIndexTarget [3] = Uni.Level3 (ti);
+ return LastIndexOfSortKey (s, start, start,
length, charSortKeyIndexTarget, target, ti, !Uni.HasSpecialWeight ((char) ti));
+ }
}
- // Searches target char w/o checking contractions
- int LastIndexOfPrimitiveChar (string s, int start, int length,
char target)
- {
- int ti = FilterOptions ((int) target);
- charSortKey [0] = Category (ti);
- charSortKey [1] = Level1 (ti);
- if (!ignoreNonSpace)
- // FIXME: pass ExtenderType
- charSortKey [2] = Level2 (ti,
ExtenderType.None);
- charSortKey [3] = Uni.Level3 (ti);
- return LastIndexOfSortKey (s, start, length,
charSortKey, target, ti, !Uni.HasSpecialWeight ((char) ti));
- }
-
// Searches target byte[] keydata
- int LastIndexOfSortKey (string s, int start, int length, byte
[] sortkey, char target, int ti, bool noLv4)
+ int LastIndexOfSortKey (string s, int start, int orgStart, int
length, byte [] sortkey, char target, int ti, bool noLv4)
{
int end = start - length;
-
- for (int idx = start; idx > end; idx--) {
+ int idx = start;
+ while (idx > end) {
int cur = idx;
- if (Matches (s, ref idx, end, ti, target,
sortkey, noLv4, true))
+ if (MatchesBackward (s, ref idx, end, orgStart,
ti, sortkey, noLv4))
return cur;
}
return -1;
@@ -1456,20 +1440,47 @@
if (tidx == target.Length)
return start;
Contraction ct = GetContraction (target, tidx,
target.Length - tidx);
- byte [] sortkey = ct != null ? ct.SortKey : null;
string replace = ct != null ? ct.Replacement : null;
+ byte [] sk = replace == null ? charSortKeyIndexTarget :
null;
+ bool noLv4 = true;
+ char tc = char.MinValue;
+ int ti = -1;
+ if (ct != null && sk != null) {
+ for (int i = 0; i < ct.SortKey.Length; i++)
+ sk [i] = ct.SortKey [i];
+ } else if (sk != null) {
+ tc = target [tidx];
+ ti = FilterOptions (target [tidx]);
+ sk [0] = Category (ti);
+ sk [1] = Level1 (ti);
+ if (!ignoreNonSpace)
+ sk [2] = Level2 (ti, ExtenderType.None);
+ sk [3] = Uni.Level3 (ti);
+ noLv4 = !Uni.HasSpecialWeight ((char) ti);
+ }
+ if (sk != null) {
+ for (tidx++; tidx < target.Length; tidx++) {
+ if (Category (target [tidx]) != 1)
+ break;
+ if (sk [2] == 0)
+ sk [2] = 2;
+ sk [2] = (byte) (sk [2] + Level2
(target [tidx], ExtenderType.None));
+ }
+ }
+
do {
int idx = 0;
- if (sortkey != null)
- idx = LastIndexOfSortKey (s, start,
length, ct.SortKey, char.MinValue, -1, true);
- else if (replace != null)
+
+ if (replace != null)
idx = LastIndexOf (s, replace, start,
length, stringSort);
else
- idx = LastIndexOfPrimitiveChar (s,
start, length, target [tidx]);
-
+ idx = LastIndexOfSortKey (s, start,
orgStart, length, sk, tc, ti, noLv4);
if (idx < 0)
return -1;
+ length -= start - idx;
+ start = idx;
+
if (IsPrefix (s, target, idx, orgStart - idx +
1, stringSort, false)) {
for (;idx < orgStart; idx++)
if (!IsIgnorable (s [idx]))
@@ -1488,62 +1499,6 @@
return -1;
}
- private bool Matches (string s, ref int idx, int end, int ti,
char target, byte [] sortkey, bool noLv4, bool lastIndexOf)
- {
- switch (char.GetUnicodeCategory (s [idx])) {
- case UnicodeCategory.PrivateUse:
- case UnicodeCategory.Surrogate:
- if (s [idx] != target)
- return false;
- return true;
- }
-
- char sc = char.MinValue;
- Contraction ct = GetContraction (s, idx, end);
- // if lv4 exists, it never matches contraction
- if (ct != null && noLv4) {
- if (lastIndexOf)
- idx -= ct.Source.Length - 1;
- else
- idx += ct.Source.Length - 1;
- if (ct.SortKey != null) {
- for (int i = 0; i < sortkey.Length; i++)
- if (ct.SortKey [i] != sortkey
[i])
- return false;
- return true;
- }
- // Here is the core of LAMESPEC
- // described at the top of the source.
- sc = ct.Replacement [0];
- }
- else
- sc = s [idx];
-
- if (sc == target)
- return true;
- int si = FilterOptions ((int) sc);
- if (Category (si) != sortkey [0] ||
- Level1 (si) != sortkey [1] ||
- // FIXME: pass ExtenderType
- !ignoreNonSpace && Level2 (si,
ExtenderType.None) != sortkey [2] ||
- Uni.Level3 (si) != sortkey [3])
- return false;
- if (noLv4 && !Uni.HasSpecialWeight ((char) si))
- return true;
- else if (noLv4)
- return false;
- if (Uni.IsJapaneseSmallLetter ((char) si) !=
- Uni.IsJapaneseSmallLetter ((char) ti) ||
- Uni.GetJapaneseDashType ((char) si) !=
- Uni.GetJapaneseDashType ((char) ti) ||
- !Uni.IsHiragana ((char) si) !=
- !Uni.IsHiragana ((char) ti) ||
- IsHalfKana ((char) si) !=
- IsHalfKana ((char) ti))
- return false;
- return true;
- }
-
private bool MatchesForward (string s, ref int idx, int end,
int ti, byte [] sortkey, bool noLv4)
{
int si = -1;
@@ -1551,8 +1506,13 @@
Contraction ct = null;
if (ext == ExtenderType.None)
ct = GetContraction (s, idx, end);
- else if (previousChar < 0)
+ else if (previousChar < 0) {
+ if (previousSortKey == null) {
+ idx++;
+ return false;
+ }
charSortKey = previousSortKey;
+ }
else
si = FilterExtender (previousChar, ext);
// if lv4 exists, it never matches contraction
@@ -1595,6 +1555,11 @@
+ Level2 (s [idx],
ExtenderType.None));
}
+ return MatchesPrimitive (charSortKey, si, ext, sortkey,
ti, noLv4);
+ }
+
+ private bool MatchesPrimitive (byte [] charSortKey, int si,
ExtenderType ext, byte [] sortkey, int ti, bool noLv4)
+ {
if (charSortKey [0] != sortkey [0] ||
charSortKey [1] != sortkey [1] ||
(!ignoreNonSpace && charSortKey [2] != sortkey
[2]) ||
@@ -1610,8 +1575,9 @@
return false;
if (Uni.IsJapaneseSmallLetter ((char) si) !=
Uni.IsJapaneseSmallLetter ((char) ti) ||
- Uni.GetJapaneseDashType ((char) si) !=
- Uni.GetJapaneseDashType ((char) ti) ||
+ ToDashTypeValue (ext) !=
+ // FIXME: we will have to specify correct value
for target
+ ToDashTypeValue (ExtenderType.None) ||
!Uni.IsHiragana ((char) si) !=
!Uni.IsHiragana ((char) ti) ||
IsHalfKana ((char) si) !=
@@ -1619,6 +1585,89 @@
return false;
return true;
}
+
+ private bool MatchesBackward (string s, ref int idx, int end,
int orgStart, int ti, byte [] sortkey, bool noLv4)
+ {
+ int cur = idx;
+ int si = -1;
+ ExtenderType ext = GetExtenderType (s [idx]);
+ // To handle extenders in source, we need to
+ // check next _primary_ character.
+ if (ext != ExtenderType.None) {
+ byte diacritical = 0;
+ for (int tmp = 0; ; tmp--) {
+ if (tmp < 0) // heading extender
+ return false;
+ if (IsIgnorable (s [tmp]))
+ continue;
+ int tmpi = FilterOptions (s [tmp]);
+ byte category = Category (tmpi);
+ if (category == 1) {
+ diacritical = Level2 (tmpi,
ExtenderType.None);
+ continue;
+ }
+ si = FilterExtender (tmpi, ext);
+ charSortKey [0] = category;
+ charSortKey [1] = Level1 (si);
+ if (!ignoreNonSpace)
+ charSortKey [2] = Level2 (si,
ext);
+ charSortKey [3] = Uni.Level3 (si);
+ if (ext != ExtenderType.Conditional &&
+ diacritical != 0)
+ charSortKey [2] =
+ (charSortKey [2] == 0) ?
+ (byte) (diacritical +
2) :
+ diacritical;
+ break;
+ }
+ idx--;
+ }
+ Contraction ct = null;
+ if (ext == ExtenderType.None)
+ ct = GetContraction (s, idx, end);
+ // if lv4 exists, it never matches contraction
+ if (ct != null) {
+ idx -= ct.Source.Length;
+ if (!noLv4)
+ return false;
+ if (ct.SortKey != null) {
+ for (int i = 0; i < sortkey.Length; i++)
+ charSortKey [i] = sortkey [i];
+ previousChar = -1;
+ previousSortKey = charSortKey;
+ } else {
+ // Here is the core of LAMESPEC
+ // described at the top of the source.
+ int dummy = ct.Replacement.Length - 1;
+ return MatchesBackward (ct.Replacement,
+ ref dummy, dummy, -1, ti,
sortkey, noLv4);
+ }
+ } else if (ext == ExtenderType.None) {
+ if (si < 0)
+ si = FilterOptions (s [idx]);
+ charSortKey [0] = Category (si);
+ charSortKey [1] = Level1 (si);
+ if (!ignoreNonSpace)
+ charSortKey [2] = Level2 (si, ext);
+ charSortKey [3] = Uni.Level3 (si);
+ if (charSortKey [0] != 1)
+ previousChar = si;
+ idx--;
+ }
+ if (ext == ExtenderType.None) {
+ for (int tmp = cur + 1; tmp < orgStart; tmp++) {
+ if (Category (s [tmp]) != 1)
+ break;
+ if (ignoreNonSpace)
+ continue;
+ if (charSortKey [2] == 0)
+ charSortKey [2] = 2;
+ charSortKey [2] = (byte) (charSortKey
[2]
+ + Level2 (s [tmp],
ExtenderType.None));
+ }
+ }
+ return MatchesPrimitive (charSortKey, si, ext, sortkey,
ti, noLv4);
+ }
#endregion
}
}
Modified: trunk/mcs/class/corlib/Mono.Globalization.Unicode/TestDriver.cs
===================================================================
--- trunk/mcs/class/corlib/Mono.Globalization.Unicode/TestDriver.cs
2005-07-15 04:55:23 UTC (rev 47326)
+++ trunk/mcs/class/corlib/Mono.Globalization.Unicode/TestDriver.cs
2005-07-15 05:37:18 UTC (rev 47327)
@@ -30,10 +30,7 @@
void Run ()
{
-//IsSuffix ("\u00E6", "ae", CompareOptions.None);
-IsSuffix ("ae", "\u00E6", CompareOptions.None);
-//LastIndexOf ("AE", "A", 1, 2, CompareOptions.None);
-//LastIndexOf ("\u00C6", "AE", CompareOptions.None);
+LastIndexOf ("\u30D1\u30FC", "\u30A2", CompareOptions.IgnoreNonSpace);
return;
/*
DumpSortKey ("AE");
Modified: trunk/mcs/class/corlib/Test/System.Globalization/ChangeLog
===================================================================
--- trunk/mcs/class/corlib/Test/System.Globalization/ChangeLog 2005-07-15
04:55:23 UTC (rev 47326)
+++ trunk/mcs/class/corlib/Test/System.Globalization/ChangeLog 2005-07-15
05:37:18 UTC (rev 47327)
@@ -1,3 +1,8 @@
+2005-07-15 Atsushi Enomoto <[EMAIL PROTECTED]>
+
+ * CompareInfoTest.cs : Uncommented tests that got working. Added two
+ cornercase notworking LastIndexOf() tests.
+
2005-07-14 Atsushi Enomoto <[EMAIL PROTECTED]>
* CompareInfoTest.cs : IsSuffix() extender tests got working.
Modified: trunk/mcs/class/corlib/Test/System.Globalization/CompareInfoTest.cs
===================================================================
--- trunk/mcs/class/corlib/Test/System.Globalization/CompareInfoTest.cs
2005-07-15 04:55:23 UTC (rev 47326)
+++ trunk/mcs/class/corlib/Test/System.Globalization/CompareInfoTest.cs
2005-07-15 05:37:18 UTC (rev 47327)
@@ -868,7 +868,6 @@
AssertIsSuffix ("#10", true, "\uff21\uff21", "\u3007\uff21",
CompareOptions.None);
AssertIsSuffix ("#11", true, "\uff21\uff21", "\uff21\u3007",
CompareOptions.None);
// extender in target
- // FIXME: not working
AssertIsSuffix ("#12", false, "\u30D1\u30A2", "\u30D1\u30FC");
AssertIsSuffix ("#13", true, "\u30D1\u30A2", "\u30D1\u30FC",
CompareOptions.IgnoreNonSpace);
// extender in source
@@ -981,17 +980,20 @@
AssertLastIndexOf ("#1-3", 0, "\u30D1\u30A2", "\u30D1\u30FC",
CompareOptions.IgnoreNonSpace);
// extender in source
AssertLastIndexOf ("#1-4", 0, "\u30D1\u30FC", "\u30D1\u30A2",
CompareOptions.IgnoreNonSpace);
- // FIXME: not working (extender support required)
-// AssertLastIndexOf ("#2", 0, "\u30D1\u30FC\u30B9",
"\uFF8A\uFF9F\uFF70\uFF7D", CompareOptions.IgnoreWidth);
- // FIXME: not working (extender support required)
-// AssertLastIndexOf ("#3", 0, "\uFF80\uFF9E\uFF72\uFF8C\uFF9E",
-// "\u30C0\u30A4\u30D6", CompareOptions.IgnoreWidth);
+ // FIXME: not working (extender support is not complete.
+ // Currently private IsPrefix() cannot handle heading
+ // extenders to consume previous primary char.)
+// AssertLastIndexOf ("#1-5", 1, "\u30D1\u30FC", "\u30A2",
CompareOptions.IgnoreNonSpace);
+ // this shows that Windows accesses beyond the length and
+ // acquires the corresponding character to expand.
+// AssertLastIndexOf ("#1-6", 1, "\u30D1\u30FC", "\u30A2", 1, 1,
CompareOptions.IgnoreNonSpace, invariant);
+ AssertLastIndexOf ("#2", 0, "\u30D1\u30FC\u30B9",
"\uFF8A\uFF9F\uFF70\uFF7D", CompareOptions.IgnoreWidth);
+ AssertLastIndexOf ("#3", 0, "\uFF80\uFF9E\uFF72\uFF8C\uFF9E",
+ "\u30C0\u30A4\u30D6", CompareOptions.IgnoreWidth);
AssertLastIndexOf ("#4", -1, "\u3042\u309D", "\u3042\u3042");
AssertLastIndexOf ("#5", 0, "\u3042\u309D", "\u3042\u3042",
CompareOptions.IgnoreNonSpace);
- // FIXME: not working (extender support required)
-// AssertLastIndexOf ("#6", 0, "\uFF8A\uFF9E\uFF70\uFF99",
-// "\u30D0\u30FC\u30EB", CompareOptions.IgnoreWidth);
-
+ AssertLastIndexOf ("#6", 0, "\uFF8A\uFF9E\uFF70\uFF99",
+ "\u30D0\u30FC\u30EB", CompareOptions.IgnoreWidth);
}
}
_______________________________________________
Mono-patches maillist - [email protected]
http://lists.ximian.com/mailman/listinfo/mono-patches
