Bruce Momjian wrote: > Alvaro Herrera wrote: > > Bruce Momjian wrote: > > > > > I moved str_initcap() over into oracle_compat.c and then had initcap() > > > convert to/from TEXT to call it. The code is a little weird because > > > str_initcap() needs to convert to text to use texttowcs(), so in > > > multibyte encodings initcap converts the string to text, then to char, > > > then to text to call texttowcs(). I didn't see a cleaner way to do > > > this. > > > > Why not use wchar2char? It seems there's room for extra cleanup here. > > > > Also, the prototype of str_initcap in builtins.h looks out of place. > > I talked to Alvaro on IM, and there is certainly much more cleanup to do > in this area. I will work from the bottom up. First, is moving the > USE_WIDE_UPPER_LOWER define to c.h, and removing TS_USE_WIDE and using > USE_WIDE_UPPER_LOWER instead. Patch attached and applied.
The second step is to move wchar2char() and char2wchar() from tsearch into /mb to be easier to use for other modules; also move pnstrdup(). Patch attached and applied. -- Bruce Momjian <[EMAIL PROTECTED]> http://momjian.us EnterpriseDB http://enterprisedb.com + If your life is a hard drive, Christ can be your backup. +
Index: src/backend/tsearch/ts_locale.c =================================================================== RCS file: /cvsroot/pgsql/src/backend/tsearch/ts_locale.c,v retrieving revision 1.8 diff -c -c -r1.8 ts_locale.c *** src/backend/tsearch/ts_locale.c 17 Jun 2008 16:09:06 -0000 1.8 --- src/backend/tsearch/ts_locale.c 18 Jun 2008 18:37:02 -0000 *************** *** 16,140 **** #include "tsearch/ts_locale.h" #include "tsearch/ts_public.h" - #ifdef USE_WIDE_UPPER_LOWER - /* - * wchar2char --- convert wide characters to multibyte format - * - * This has the same API as the standard wcstombs() function; in particular, - * tolen is the maximum number of bytes to store at *to, and *from must be - * zero-terminated. The output will be zero-terminated iff there is room. - */ - size_t - wchar2char(char *to, const wchar_t *from, size_t tolen) - { - if (tolen == 0) - return 0; - - #ifdef WIN32 - if (GetDatabaseEncoding() == PG_UTF8) - { - int r; - - r = WideCharToMultiByte(CP_UTF8, 0, from, -1, to, tolen, - NULL, NULL); - - if (r <= 0) - return (size_t) -1; - - Assert(r <= tolen); - - /* Microsoft counts the zero terminator in the result */ - return r - 1; - } - #endif /* WIN32 */ - - return wcstombs(to, from, tolen); - } - - /* - * char2wchar --- convert multibyte characters to wide characters - * - * This has almost the API of mbstowcs(), except that *from need not be - * null-terminated; instead, the number of input bytes is specified as - * fromlen. Also, we ereport() rather than returning -1 for invalid - * input encoding. tolen is the maximum number of wchar_t's to store at *to. - * The output will be zero-terminated iff there is room. - */ - size_t - char2wchar(wchar_t *to, size_t tolen, const char *from, size_t fromlen) - { - if (tolen == 0) - return 0; - - #ifdef WIN32 - if (GetDatabaseEncoding() == PG_UTF8) - { - int r; - - /* stupid Microsloth API does not work for zero-length input */ - if (fromlen == 0) - r = 0; - else - { - r = MultiByteToWideChar(CP_UTF8, 0, from, fromlen, to, tolen - 1); - - if (r <= 0) - { - /* see notes in oracle_compat.c about error reporting */ - pg_verifymbstr(from, fromlen, false); - ereport(ERROR, - (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE), - errmsg("invalid multibyte character for locale"), - errhint("The server's LC_CTYPE locale is probably incompatible with the database encoding."))); - } - } - - Assert(r < tolen); - to[r] = 0; - - return r; - } - #endif /* WIN32 */ - - if (lc_ctype_is_c()) - { - /* - * pg_mb2wchar_with_len always adds trailing '\0', so 'to' should be - * allocated with sufficient space - */ - return pg_mb2wchar_with_len(from, (pg_wchar *) to, fromlen); - } - else - { - /* - * mbstowcs requires ending '\0' - */ - char *str = pnstrdup(from, fromlen); - size_t result; - - result = mbstowcs(to, str, tolen); - - pfree(str); - - if (result == (size_t) -1) - { - pg_verifymbstr(from, fromlen, false); - ereport(ERROR, - (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE), - errmsg("invalid multibyte character for locale"), - errhint("The server's LC_CTYPE locale is probably incompatible with the database encoding."))); - } - - if (result < tolen) - to[result] = 0; - - return result; - } - } - - int t_isdigit(const char *ptr) { --- 16,23 ---- Index: src/backend/tsearch/ts_utils.c =================================================================== RCS file: /cvsroot/pgsql/src/backend/tsearch/ts_utils.c,v retrieving revision 1.9 diff -c -c -r1.9 ts_utils.c *** src/backend/tsearch/ts_utils.c 1 Jan 2008 19:45:52 -0000 1.9 --- src/backend/tsearch/ts_utils.c 18 Jun 2008 18:37:02 -0000 *************** *** 153,165 **** bsearch(&key, s->stop, s->len, sizeof(char *), comparestr)) ? true : false; } - - char * - pnstrdup(const char *in, int len) - { - char *out = palloc(len + 1); - - memcpy(out, in, len); - out[len] = '\0'; - return out; - } --- 153,155 ---- Index: src/backend/utils/mb/mbutils.c =================================================================== RCS file: /cvsroot/pgsql/src/backend/utils/mb/mbutils.c,v retrieving revision 1.71 diff -c -c -r1.71 mbutils.c *** src/backend/utils/mb/mbutils.c 27 May 2008 12:24:42 -0000 1.71 --- src/backend/utils/mb/mbutils.c 18 Jun 2008 18:37:02 -0000 *************** *** 555,560 **** --- 555,688 ---- return result; } + + + #ifdef USE_WIDE_UPPER_LOWER + + /* + * wchar2char --- convert wide characters to multibyte format + * + * This has the same API as the standard wcstombs() function; in particular, + * tolen is the maximum number of bytes to store at *to, and *from must be + * zero-terminated. The output will be zero-terminated iff there is room. + */ + size_t + wchar2char(char *to, const wchar_t *from, size_t tolen) + { + size_t result; + + if (tolen == 0) + return 0; + + #ifdef WIN32 + /* + * On Windows, the "Unicode" locales assume UTF16 not UTF8 encoding, + * and for some reason mbstowcs and wcstombs won't do this for us, + * so we use MultiByteToWideChar(). + */ + if (GetDatabaseEncoding() == PG_UTF8) + { + result = WideCharToMultiByte(CP_UTF8, 0, from, -1, to, tolen, + NULL, NULL); + /* A zero return is failure */ + if (result <= 0) + result = -1; + else + { + Assert(result <= tolen); + /* Microsoft counts the zero terminator in the result */ + result--; + } + } + else + #endif /* WIN32 */ + result = wcstombs(to, from, tolen); + return result; + } + + /* + * char2wchar --- convert multibyte characters to wide characters + * + * This has almost the API of mbstowcs(), except that *from need not be + * null-terminated; instead, the number of input bytes is specified as + * fromlen. Also, we ereport() rather than returning -1 for invalid + * input encoding. tolen is the maximum number of wchar_t's to store at *to. + * The output will be zero-terminated iff there is room. + */ + size_t + char2wchar(wchar_t *to, size_t tolen, const char *from, size_t fromlen) + { + size_t result; + + if (tolen == 0) + return 0; + + #ifdef WIN32 + /* See WIN32 "Unicode" comment above */ + if (GetDatabaseEncoding() == PG_UTF8) + { + /* Win32 API does not work for zero-length input */ + if (fromlen == 0) + result = 0; + else + { + result = MultiByteToWideChar(CP_UTF8, 0, from, fromlen, to, tolen - 1); + /* A zero return is failure */ + if (result == 0) + result = -1; + } + + if (result != -1) + { + Assert(result < tolen); + /* Append trailing null wchar (MultiByteToWideChar() does not) */ + to[result] = 0; + } + } + else + #endif /* WIN32 */ + { + if (lc_ctype_is_c()) + { + /* + * pg_mb2wchar_with_len always adds trailing '\0', so 'to' should be + * allocated with sufficient space + */ + result = pg_mb2wchar_with_len(from, (pg_wchar *) to, fromlen); + } + else + { + /* mbstowcs requires ending '\0' */ + char *str = pnstrdup(from, fromlen); + + result = mbstowcs(to, str, tolen); + pfree(str); + } + } + + if (result == -1) + { + /* + * Invalid multibyte character encountered. We try to give a useful + * error message by letting pg_verifymbstr check the string. But it's + * possible that the string is OK to us, and not OK to mbstowcs --- + * this suggests that the LC_CTYPE locale is different from the + * database encoding. Give a generic error message if verifymbstr + * can't find anything wrong. + */ + pg_verifymbstr(from, fromlen, false); /* might not return */ + /* but if it does ... */ + ereport(ERROR, + (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE), + errmsg("invalid multibyte character for locale"), + errhint("The server's LC_CTYPE locale is probably incompatible with the database encoding."))); + } + + return result; + } + + #endif + /* convert a multibyte string to a wchar */ int pg_mb2wchar(const char *from, pg_wchar *to) Index: src/backend/utils/mmgr/mcxt.c =================================================================== RCS file: /cvsroot/pgsql/src/backend/utils/mmgr/mcxt.c,v retrieving revision 1.63 diff -c -c -r1.63 mcxt.c *** src/backend/utils/mmgr/mcxt.c 1 Jan 2008 19:45:55 -0000 1.63 --- src/backend/utils/mmgr/mcxt.c 18 Jun 2008 18:37:05 -0000 *************** *** 624,629 **** --- 624,641 ---- pointer, size); } + /* Like pstrdup(), but append null byte */ + char * + pnstrdup(const char *in, int len) + { + char *out = palloc(len + 1); + + memcpy(out, in, len); + out[len] = '\0'; + return out; + } + + /* * MemoryContextSwitchTo * Returns the current context; installs the given context. Index: src/include/mb/pg_wchar.h =================================================================== RCS file: /cvsroot/pgsql/src/include/mb/pg_wchar.h,v retrieving revision 1.78 diff -c -c -r1.78 pg_wchar.h *** src/include/mb/pg_wchar.h 1 Jan 2008 19:45:58 -0000 1.78 --- src/include/mb/pg_wchar.h 18 Jun 2008 18:37:05 -0000 *************** *** 362,367 **** --- 362,372 ---- extern int pg_encoding_max_length(int encoding); extern int pg_database_encoding_max_length(void); + #ifdef USE_WIDE_UPPER_LOWER + extern size_t wchar2char(char *to, const wchar_t *from, size_t tolen); + extern size_t char2wchar(wchar_t *to, size_t tolen, const char *from, size_t fromlen); + #endif + extern void SetDefaultClientEncoding(void); extern int SetClientEncoding(int encoding, bool doit); extern void InitializeClientEncoding(void); Index: src/include/tsearch/ts_locale.h =================================================================== RCS file: /cvsroot/pgsql/src/include/tsearch/ts_locale.h,v retrieving revision 1.6 diff -c -c -r1.6 ts_locale.h *** src/include/tsearch/ts_locale.h 17 Jun 2008 16:09:06 -0000 1.6 --- src/include/tsearch/ts_locale.h 18 Jun 2008 18:37:05 -0000 *************** *** 33,41 **** #ifdef USE_WIDE_UPPER_LOWER - extern size_t wchar2char(char *to, const wchar_t *from, size_t tolen); - extern size_t char2wchar(wchar_t *to, size_t tolen, const char *from, size_t fromlen); - extern int t_isdigit(const char *ptr); extern int t_isspace(const char *ptr); extern int t_isalpha(const char *ptr); --- 33,38 ---- Index: src/include/tsearch/ts_public.h =================================================================== RCS file: /cvsroot/pgsql/src/include/tsearch/ts_public.h,v retrieving revision 1.9 diff -c -c -r1.9 ts_public.h *** src/include/tsearch/ts_public.h 16 May 2008 16:31:02 -0000 1.9 --- src/include/tsearch/ts_public.h 18 Jun 2008 18:37:05 -0000 *************** *** 62,69 **** extern char *get_tsearch_config_filename(const char *basename, const char *extension); - extern char *pnstrdup(const char *in, int len); - /* * Often useful stopword list management */ --- 62,67 ---- Index: src/include/utils/palloc.h =================================================================== RCS file: /cvsroot/pgsql/src/include/utils/palloc.h,v retrieving revision 1.38 diff -c -c -r1.38 palloc.h *** src/include/utils/palloc.h 1 Jan 2008 19:45:59 -0000 1.38 --- src/include/utils/palloc.h 18 Jun 2008 18:37:05 -0000 *************** *** 70,75 **** --- 70,77 ---- extern void *repalloc(void *pointer, Size size); + extern char *pnstrdup(const char *in, int len); + /* * MemoryContextSwitchTo can't be a macro in standard C compilers. * But we can make it an inline function when using GCC.
-- Sent via pgsql-patches mailing list (pgsql-patches@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-patches