Author: miguel
Date: 2006-10-06 16:30:29 -0400 (Fri, 06 Oct 2006)
New Revision: 66375
Modified:
trunk/mono/eglib/ChangeLog
trunk/mono/eglib/TODO
trunk/mono/eglib/src/glib.h
trunk/mono/eglib/src/gstr.c
trunk/mono/eglib/src/gunicode.c
trunk/mono/eglib/test/utf8.c
Log:
2006-10-06 Miguel de Icaza <[EMAIL PROTECTED]>
* src/gunicode.c (g_convert): Add bytes_read, bytes_written
support; Small fixes to avoid valgrind errors.
(g_utf8_validate): Add, based on the libxml code.
gstr.c: implement g_strdown.
Modified: trunk/mono/eglib/ChangeLog
===================================================================
--- trunk/mono/eglib/ChangeLog 2006-10-06 20:11:10 UTC (rev 66374)
+++ trunk/mono/eglib/ChangeLog 2006-10-06 20:30:29 UTC (rev 66375)
@@ -1,3 +1,12 @@
+2006-10-06 Miguel de Icaza <[EMAIL PROTECTED]>
+
+ * src/gunicode.c (g_convert): Add bytes_read, bytes_written
+ support; Small fixes to avoid valgrind errors.
+
+ (g_utf8_validate): Add, based on the libxml code.
+
+ gstr.c: implement g_strdown.
+
2006-10-05 Miguel de Icaza <[EMAIL PROTECTED]>
* src/gunicode.c: Some work in progress to implement g_convert
Modified: trunk/mono/eglib/TODO
===================================================================
--- trunk/mono/eglib/TODO 2006-10-06 20:11:10 UTC (rev 66374)
+++ trunk/mono/eglib/TODO 2006-10-06 20:30:29 UTC (rev 66375)
@@ -25,16 +25,10 @@
* String manipulation
1 g_filename_from_utf8 [LIMITATION: UTF8 only today]
1 g_ascii_strdown
- 4 g_strdown
10 g_strlcpy
2 g_strdelimit
1 g_strescape
- * Character encoding
- 3 g_utf8_validate
- 3 g_locale_to_utf8
- 1 g_locale_from_utf8
-
* Miscelaneous
3 g_spaced_primes_closest
2 g_printerr
@@ -56,7 +50,6 @@
* Unimplemented, not supported currently:
- 3 g_convert (used for MONO_EXTERNAL_ENCODINGS)
g_unichar_tolower Used for deprecated unmanaged string
collation
g_unichar_type Used for deprecated unmanaged string
collation
Modified: trunk/mono/eglib/src/glib.h
===================================================================
--- trunk/mono/eglib/src/glib.h 2006-10-06 20:11:10 UTC (rev 66374)
+++ trunk/mono/eglib/src/glib.h 2006-10-06 20:30:29 UTC (rev 66375)
@@ -651,11 +651,14 @@
gchar *g_locale_to_utf8 (const gchar *opsysstring, gssize len,
gsize *bytes_read, gsize *bytes_written,
GError **error);
+gchar *g_locale_from_utf8 (const gchar *utf8string, gssize len, gsize
*bytes_read,
+ gsize *bytes_written, GError **error);
gchar *g_filename_from_utf8 (const gchar *utf8string, gssize len, gsize
*bytes_read,
gsize *bytes_written, GError **error);
gchar *g_convert (const gchar *str, gssize len,
const gchar *to_codeset, const gchar
*from_codeset,
gsize *bytes_read, gsize *bytes_written, GError
**error);
+gboolean g_utf8_validate (const gchar *str, gssize max_len, const gchar
**end);
#endif
Modified: trunk/mono/eglib/src/gstr.c
===================================================================
--- trunk/mono/eglib/src/gstr.c 2006-10-06 20:11:10 UTC (rev 66374)
+++ trunk/mono/eglib/src/gstr.c 2006-10-06 20:30:29 UTC (rev 66375)
@@ -475,3 +475,13 @@
}
return result;
}
+
+void
+g_strdown (gchar *string)
+{
+ g_return_if_fail (string != NULL);
+
+ while (*string){
+ *string = tolower (*string);
+ }
+}
Modified: trunk/mono/eglib/src/gunicode.c
===================================================================
--- trunk/mono/eglib/src/gunicode.c 2006-10-06 20:11:10 UTC (rev 66374)
+++ trunk/mono/eglib/src/gunicode.c 2006-10-06 20:30:29 UTC (rev 66375)
@@ -6,6 +6,13 @@
*
* (C) 2006 Novell, Inc.
*
+ * utf8 validation code came from:
+ * libxml2-2.6.26 licensed under the MIT X11 license
+ *
+ * Authors credit in libxml's string.c:
+ * William Brack <[EMAIL PROTECTED]>
+ * [EMAIL PROTECTED]
+ *
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
@@ -24,6 +31,7 @@
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
*/
#include <stdio.h>
#include <glib.h>
@@ -55,6 +63,7 @@
{
iconv_t convertor;
char *buffer, *result, *output;
+ const char *strptr = (const char *) str;
int str_len = len == -1 ? strlen (str) : len;
int buffer_size;
size_t left, out_left;
@@ -66,19 +75,21 @@
return NULL;
}
- buffer_size = out_left = str_len + 1;
- buffer = g_malloc (out_left);
+ buffer_size = str_len + 1 + 8;
+ buffer = g_malloc (buffer_size);
+ out_left = str_len;
output = buffer;
left = str_len;
while (left > 0){
- int res = iconv (convertor, (char **) &str, &left, &output,
&out_left);
+ int res = iconv (convertor, (char **) &strptr, &left, &output,
&out_left);
if (res == (size_t) -1){
- int out_size = buffer_size - out_left;
-
if (errno == E2BIG){
char *n;
-
- buffer_size += left + 2;
+ int extra_space = 8 + left;
+ int output_used = output - buffer;
+
+ buffer_size += extra_space;
+
n = g_realloc (buffer, buffer_size);
if (n == NULL){
@@ -89,8 +100,8 @@
goto leave;
}
buffer = n;
- output = buffer + out_size;
- out_left = buffer_size - out_size;
+ out_left += extra_space;
+ output = buffer + output_used;
} else if (errno == EILSEQ){
if (error != NULL)
*error = g_error_new (NULL,
G_CONVERT_ERROR_ILLEGAL_SEQUENCE, "Invalid multi-byte sequence on input");
@@ -106,6 +117,10 @@
}
}
}
+ if (bytes_read != NULL)
+ *bytes_read = strptr - str;
+ if (bytes_written != NULL)
+ *bytes_written = output - buffer;
*output = 0;
result = buffer;
leave:
@@ -151,3 +166,82 @@
return g_convert (opsysstring, len, "UTF-8", my_charset, bytes_read,
bytes_written, error);
}
+gchar *
+g_locale_from_utf8 (const gchar *utf8string, gssize len, gsize *bytes_read,
gsize *bytes_written, GError **error)
+{
+ g_get_charset (NULL);
+
+ return g_convert (utf8string, len, my_charset, "UTF-8", bytes_read,
bytes_written, error);
+}
+/**
+ * g_utf8_validate
+ * @utf: Pointer to putative UTF-8 encoded string.
+ *
+ * Checks @utf for being valid UTF-8. @utf is assumed to be
+ * null-terminated. This function is not super-strict, as it will
+ * allow longer UTF-8 sequences than necessary. Note that Java is
+ * capable of producing these sequences if provoked. Also note, this
+ * routine checks for the 4-byte maximum size, but does not check for
+ * 0x10ffff maximum value.
+ *
+ * Return value: true if @utf is valid.
+ **/
+gboolean
+g_utf8_validate (const gchar *utf, gssize max_len, const gchar **end)
+{
+ int ix;
+
+ g_return_val_if_fail (utf != NULL, FALSE);
+
+ if (max_len == -1)
+ max_len = strlen (utf);
+
+ /*
+ * utf is a string of 1, 2, 3 or 4 bytes. The valid strings
+ * are as follows (in "bit format"):
+ * 0xxxxxxx valid 1-byte
+ * 110xxxxx 10xxxxxx valid 2-byte
+ * 1110xxxx 10xxxxxx 10xxxxxx valid 3-byte
+ * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx valid 4-byte
+ */
+ for (ix = 0; ix < max_len;) { /* string is 0-terminated */
+ unsigned char c;
+
+ c = utf[ix];
+ if ((c & 0x80) == 0x00) { /* 1-byte code, starts with 10
*/
+ ix++;
+ } else if ((c & 0xe0) == 0xc0) {/* 2-byte code, starts with 110
*/
+ if (((ix+1) >= max_len) || (utf[ix+1] & 0xc0 ) != 0x80){
+ if (end != NULL)
+ *end = &utf [ix];
+ return FALSE;
+ }
+ ix += 2;
+ } else if ((c & 0xf0) == 0xe0) {/* 3-byte code, starts with
1110 */
+ if (((ix + 2) >= max_len) ||
+ ((utf[ix+1] & 0xc0) != 0x80) ||
+ ((utf[ix+2] & 0xc0) != 0x80)){
+ if (end != NULL)
+ *end = &utf [ix];
+ return FALSE;
+ }
+ ix += 3;
+ } else if ((c & 0xf8) == 0xf0) {/* 4-byte code, starts with
11110 */
+ if (((ix + 3) >= max_len) ||
+ ((utf[ix+1] & 0xc0) != 0x80) ||
+ ((utf[ix+2] & 0xc0) != 0x80) ||
+ ((utf[ix+3] & 0xc0) != 0x80)){
+ if (end != NULL)
+ *end = &utf [ix];
+ return FALSE;
+ }
+ ix += 4;
+ } else {/* unknown encoding */
+ if (end != NULL)
+ *end = &utf [ix];
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
Modified: trunk/mono/eglib/test/utf8.c
===================================================================
--- trunk/mono/eglib/test/utf8.c 2006-10-06 20:11:10 UTC (rev 66374)
+++ trunk/mono/eglib/test/utf8.c 2006-10-06 20:30:29 UTC (rev 66375)
@@ -197,9 +197,19 @@
test_convert ()
{
gsize n;
- char *s = g_convert ("\242", -1, "UTF-8", "ISO-8859-1", NULL, &n, NULL);
+ char *s = g_convert
("\242\241\243\242\241\243\242\241\243\242\241\243", -1, "UTF-8", "ISO-8859-1",
NULL, &n, NULL);
+ guchar *u = (guchar *) s;
+
+ if (strlen (s) != 24)
+ return FAILED ("Expected 24 bytes, got: %d", strlen (s));
- printf ("Result: %s %d\n", s, strlen (s));
+ if (u [1] != 162 || u [2] != 194 ||
+ u [3] != 161 || u [4] != 194 ||
+ u [5] != 163 || u [6] != 194)
+ return FAILED ("Incorrect conversion");
+
+ g_free (s);
+
return OK;
}
_______________________________________________
Mono-patches maillist - [email protected]
http://lists.ximian.com/mailman/listinfo/mono-patches
