Author: Armin Rigo <[email protected]>
Branch:
Changeset: r44076:f4ded53c2b57
Date: 2011-05-11 16:05 +0200
http://bitbucket.org/pypy/pypy/changeset/f4ded53c2b57/
Log: merge heads
diff --git a/pypy/module/_multibytecodec/c_codecs.py
b/pypy/module/_multibytecodec/c_codecs.py
--- a/pypy/module/_multibytecodec/c_codecs.py
+++ b/pypy/module/_multibytecodec/c_codecs.py
@@ -16,19 +16,20 @@
return 'EncodeDecodeError(%r, %r, %r)' % (self.start, self.end,
self.reason)
-
-srcdir = py.path.local(pypydir).join('module', '_multibytecodec', 'cjkcodecs')
+srcdir = py.path.local(pypydir).join('translator', 'c')
eci = ExternalCompilationInfo(
separate_module_files = [
- srcdir.join('_codecs_cn.c'),
- srcdir.join('_codecs_hk.c'),
- srcdir.join('_codecs_iso2022.c'),
- srcdir.join('_codecs_jp.c'),
- srcdir.join('_codecs_kr.c'),
- srcdir.join('_codecs_tw.c'),
- srcdir.join('multibytecodec.c'),
+ srcdir.join('src', 'cjkcodecs', '_codecs_cn.c'),
+ srcdir.join('src', 'cjkcodecs', '_codecs_hk.c'),
+ srcdir.join('src', 'cjkcodecs', '_codecs_iso2022.c'),
+ srcdir.join('src', 'cjkcodecs', '_codecs_jp.c'),
+ srcdir.join('src', 'cjkcodecs', '_codecs_kr.c'),
+ srcdir.join('src', 'cjkcodecs', '_codecs_tw.c'),
+ srcdir.join('src', 'cjkcodecs', 'multibytecodec.c'),
],
+ includes = ['src/cjkcodecs/multibytecodec.h'],
+ include_dirs = [str(srcdir)],
)
MBERR_TOOSMALL = -1 # insufficient output buffer space
@@ -147,9 +148,11 @@
src = rffi.cast_ptr_to_adr(src) + rffi.itemoffsetof(rffi.CWCHARP.TO)
rffi.raw_memcopy(src, dest,
llmemory.sizeof(lltype.UniChar) * length)
- return hlunicode(result)
+ got = hlunicode(result)
finally:
keepalive_until_here(result)
+ assert got is not None
+ return got
# ____________________________________________________________
# Encoding
@@ -227,6 +230,8 @@
src = rffi.cast_ptr_to_adr(src) + rffi.itemoffsetof(rffi.CCHARP.TO)
rffi.raw_memcopy(src, dest,
llmemory.sizeof(lltype.Char) * length)
- return hlstr(result)
+ got = hlstr(result)
finally:
keepalive_until_here(result)
+ assert got is not None
+ return got
diff --git a/pypy/module/_multibytecodec/test/test_translation.py
b/pypy/module/_multibytecodec/test/test_translation.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_multibytecodec/test/test_translation.py
@@ -0,0 +1,19 @@
+from pypy.module._multibytecodec import c_codecs
+from pypy.translator.c.test import test_standalone
+
+
+class TestTranslation(test_standalone.StandaloneTests):
+
+ def test_translation(self):
+ #
+ def entry_point(argv):
+ codecname, string = argv[1], argv[2]
+ c = c_codecs.getcodec(codecname)
+ u = c_codecs.decode(c, string)
+ r = c_codecs.encode(c, u)
+ print r
+ return 0
+ #
+ t, cbuilder = self.compile(entry_point)
+ data = cbuilder.cmdexec('hz \~\{abc\}')
+ assert data == '~{abc}~}\n'
diff --git a/pypy/module/_multibytecodec/cjkcodecs/README
b/pypy/translator/c/src/cjkcodecs/README
rename from pypy/module/_multibytecodec/cjkcodecs/README
rename to pypy/translator/c/src/cjkcodecs/README
diff --git a/pypy/module/_multibytecodec/cjkcodecs/_codecs_cn.c
b/pypy/translator/c/src/cjkcodecs/_codecs_cn.c
rename from pypy/module/_multibytecodec/cjkcodecs/_codecs_cn.c
rename to pypy/translator/c/src/cjkcodecs/_codecs_cn.c
--- a/pypy/module/_multibytecodec/cjkcodecs/_codecs_cn.c
+++ b/pypy/translator/c/src/cjkcodecs/_codecs_cn.c
@@ -4,8 +4,8 @@
* Written by Hye-Shik Chang <[email protected]>
*/
-#include "cjkcodecs.h"
-#include "mappings_cn.h"
+#include "src/cjkcodecs/cjkcodecs.h"
+#include "src/cjkcodecs/mappings_cn.h"
/**
* hz is predefined as 100 on AIX. So we undefine it to avoid
diff --git a/pypy/module/_multibytecodec/cjkcodecs/_codecs_hk.c
b/pypy/translator/c/src/cjkcodecs/_codecs_hk.c
rename from pypy/module/_multibytecodec/cjkcodecs/_codecs_hk.c
rename to pypy/translator/c/src/cjkcodecs/_codecs_hk.c
--- a/pypy/module/_multibytecodec/cjkcodecs/_codecs_hk.c
+++ b/pypy/translator/c/src/cjkcodecs/_codecs_hk.c
@@ -6,8 +6,8 @@
#define USING_IMPORTED_MAPS
-#include "cjkcodecs.h"
-#include "mappings_hk.h"
+#include "src/cjkcodecs/cjkcodecs.h"
+#include "src/cjkcodecs/mappings_hk.h"
/*
* BIG5HKSCS codec
diff --git a/pypy/module/_multibytecodec/cjkcodecs/_codecs_iso2022.c
b/pypy/translator/c/src/cjkcodecs/_codecs_iso2022.c
rename from pypy/module/_multibytecodec/cjkcodecs/_codecs_iso2022.c
rename to pypy/translator/c/src/cjkcodecs/_codecs_iso2022.c
--- a/pypy/module/_multibytecodec/cjkcodecs/_codecs_iso2022.c
+++ b/pypy/translator/c/src/cjkcodecs/_codecs_iso2022.c
@@ -10,10 +10,10 @@
#define EMULATE_JISX0213_2000_ENCODE_INVALID MAP_UNMAPPABLE
#define EMULATE_JISX0213_2000_DECODE_INVALID MAP_UNMAPPABLE
-#include "cjkcodecs.h"
-#include "alg_jisx0201.h"
-#include "emu_jisx0213_2000.h"
-#include "mappings_jisx0213_pair.h"
+#include "src/cjkcodecs/cjkcodecs.h"
+#include "src/cjkcodecs/alg_jisx0201.h"
+#include "src/cjkcodecs/emu_jisx0213_2000.h"
+#include "src/cjkcodecs/mappings_jisx0213_pair.h"
/* STATE
diff --git a/pypy/module/_multibytecodec/cjkcodecs/_codecs_jp.c
b/pypy/translator/c/src/cjkcodecs/_codecs_jp.c
rename from pypy/module/_multibytecodec/cjkcodecs/_codecs_jp.c
rename to pypy/translator/c/src/cjkcodecs/_codecs_jp.c
--- a/pypy/module/_multibytecodec/cjkcodecs/_codecs_jp.c
+++ b/pypy/translator/c/src/cjkcodecs/_codecs_jp.c
@@ -7,11 +7,11 @@
#define USING_BINARY_PAIR_SEARCH
#define EMPBASE 0x20000
-#include "cjkcodecs.h"
-#include "mappings_jp.h"
-#include "mappings_jisx0213_pair.h"
-#include "alg_jisx0201.h"
-#include "emu_jisx0213_2000.h"
+#include "src/cjkcodecs/cjkcodecs.h"
+#include "src/cjkcodecs/mappings_jp.h"
+#include "src/cjkcodecs/mappings_jisx0213_pair.h"
+#include "src/cjkcodecs/alg_jisx0201.h"
+#include "src/cjkcodecs/emu_jisx0213_2000.h"
/*
* CP932 codec
diff --git a/pypy/module/_multibytecodec/cjkcodecs/_codecs_kr.c
b/pypy/translator/c/src/cjkcodecs/_codecs_kr.c
rename from pypy/module/_multibytecodec/cjkcodecs/_codecs_kr.c
rename to pypy/translator/c/src/cjkcodecs/_codecs_kr.c
--- a/pypy/module/_multibytecodec/cjkcodecs/_codecs_kr.c
+++ b/pypy/translator/c/src/cjkcodecs/_codecs_kr.c
@@ -4,8 +4,8 @@
* Written by Hye-Shik Chang <[email protected]>
*/
-#include "cjkcodecs.h"
-#include "mappings_kr.h"
+#include "src/cjkcodecs/cjkcodecs.h"
+#include "src/cjkcodecs/mappings_kr.h"
/*
* EUC-KR codec
diff --git a/pypy/module/_multibytecodec/cjkcodecs/_codecs_tw.c
b/pypy/translator/c/src/cjkcodecs/_codecs_tw.c
rename from pypy/module/_multibytecodec/cjkcodecs/_codecs_tw.c
rename to pypy/translator/c/src/cjkcodecs/_codecs_tw.c
--- a/pypy/module/_multibytecodec/cjkcodecs/_codecs_tw.c
+++ b/pypy/translator/c/src/cjkcodecs/_codecs_tw.c
@@ -4,8 +4,8 @@
* Written by Hye-Shik Chang <[email protected]>
*/
-#include "cjkcodecs.h"
-#include "mappings_tw.h"
+#include "src/cjkcodecs/cjkcodecs.h"
+#include "src/cjkcodecs/mappings_tw.h"
/*
* BIG5 codec
diff --git a/pypy/module/_multibytecodec/cjkcodecs/alg_jisx0201.h
b/pypy/translator/c/src/cjkcodecs/alg_jisx0201.h
rename from pypy/module/_multibytecodec/cjkcodecs/alg_jisx0201.h
rename to pypy/translator/c/src/cjkcodecs/alg_jisx0201.h
diff --git a/pypy/module/_multibytecodec/cjkcodecs/cjkcodecs.h
b/pypy/translator/c/src/cjkcodecs/cjkcodecs.h
rename from pypy/module/_multibytecodec/cjkcodecs/cjkcodecs.h
rename to pypy/translator/c/src/cjkcodecs/cjkcodecs.h
--- a/pypy/module/_multibytecodec/cjkcodecs/cjkcodecs.h
+++ b/pypy/translator/c/src/cjkcodecs/cjkcodecs.h
@@ -9,7 +9,7 @@
#ifndef _CJKCODECS_H_
#define _CJKCODECS_H_
-#include "multibytecodec.h"
+#include "src/cjkcodecs/multibytecodec.h"
/* a unicode "undefined" codepoint */
diff --git a/pypy/module/_multibytecodec/cjkcodecs/emu_jisx0213_2000.h
b/pypy/translator/c/src/cjkcodecs/emu_jisx0213_2000.h
rename from pypy/module/_multibytecodec/cjkcodecs/emu_jisx0213_2000.h
rename to pypy/translator/c/src/cjkcodecs/emu_jisx0213_2000.h
diff --git a/pypy/module/_multibytecodec/cjkcodecs/mappings_cn.h
b/pypy/translator/c/src/cjkcodecs/mappings_cn.h
rename from pypy/module/_multibytecodec/cjkcodecs/mappings_cn.h
rename to pypy/translator/c/src/cjkcodecs/mappings_cn.h
diff --git a/pypy/module/_multibytecodec/cjkcodecs/mappings_hk.h
b/pypy/translator/c/src/cjkcodecs/mappings_hk.h
rename from pypy/module/_multibytecodec/cjkcodecs/mappings_hk.h
rename to pypy/translator/c/src/cjkcodecs/mappings_hk.h
diff --git a/pypy/module/_multibytecodec/cjkcodecs/mappings_jisx0213_pair.h
b/pypy/translator/c/src/cjkcodecs/mappings_jisx0213_pair.h
rename from pypy/module/_multibytecodec/cjkcodecs/mappings_jisx0213_pair.h
rename to pypy/translator/c/src/cjkcodecs/mappings_jisx0213_pair.h
diff --git a/pypy/module/_multibytecodec/cjkcodecs/mappings_jp.h
b/pypy/translator/c/src/cjkcodecs/mappings_jp.h
rename from pypy/module/_multibytecodec/cjkcodecs/mappings_jp.h
rename to pypy/translator/c/src/cjkcodecs/mappings_jp.h
diff --git a/pypy/module/_multibytecodec/cjkcodecs/mappings_kr.h
b/pypy/translator/c/src/cjkcodecs/mappings_kr.h
rename from pypy/module/_multibytecodec/cjkcodecs/mappings_kr.h
rename to pypy/translator/c/src/cjkcodecs/mappings_kr.h
diff --git a/pypy/module/_multibytecodec/cjkcodecs/mappings_tw.h
b/pypy/translator/c/src/cjkcodecs/mappings_tw.h
rename from pypy/module/_multibytecodec/cjkcodecs/mappings_tw.h
rename to pypy/translator/c/src/cjkcodecs/mappings_tw.h
diff --git a/pypy/module/_multibytecodec/cjkcodecs/multibytecodec.c
b/pypy/translator/c/src/cjkcodecs/multibytecodec.c
rename from pypy/module/_multibytecodec/cjkcodecs/multibytecodec.c
rename to pypy/translator/c/src/cjkcodecs/multibytecodec.c
--- a/pypy/module/_multibytecodec/cjkcodecs/multibytecodec.c
+++ b/pypy/translator/c/src/cjkcodecs/multibytecodec.c
@@ -1,5 +1,5 @@
#include <stdlib.h>
-#include "multibytecodec.h"
+#include "src/cjkcodecs/multibytecodec.h"
struct pypy_cjk_dec_s *pypy_cjk_dec_init(const MultibyteCodec *codec,
diff --git a/pypy/module/_multibytecodec/cjkcodecs/multibytecodec.h
b/pypy/translator/c/src/cjkcodecs/multibytecodec.h
rename from pypy/module/_multibytecodec/cjkcodecs/multibytecodec.h
rename to pypy/translator/c/src/cjkcodecs/multibytecodec.h
--- a/pypy/module/_multibytecodec/cjkcodecs/multibytecodec.h
+++ b/pypy/translator/c/src/cjkcodecs/multibytecodec.h
@@ -8,11 +8,15 @@
#include <unistd.h>
#include <assert.h>
+#ifndef Py_UNICODE_SIZE
#define Py_UNICODE_SIZE 4
-typedef uint32_t ucs4_t, Py_UNICODE;
-typedef uint16_t ucs2_t, DBCHAR;
+typedef uint32_t Py_UNICODE;
typedef ssize_t Py_ssize_t;
#define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t) -1) >> 1))
+#endif
+
+typedef uint32_t ucs4_t;
+typedef uint16_t ucs2_t, DBCHAR;
typedef union {
@@ -99,4 +103,46 @@
Py_ssize_t pypy_cjk_enc_inbuf_remaining(struct pypy_cjk_enc_s *d);
Py_ssize_t pypy_cjk_enc_inbuf_consumed(struct pypy_cjk_enc_s* d);
+/* list of codecs defined in the .c files */
+
+#define DEFINE_CODEC(name) \
+ const MultibyteCodec *pypy_cjkcodec_##name(void);
+
+// _codecs_cn
+DEFINE_CODEC(gb2312)
+DEFINE_CODEC(gbk)
+DEFINE_CODEC(gb18030)
+DEFINE_CODEC(hz)
+
+//_codecs_hk
+DEFINE_CODEC(big5hkscs)
+
+//_codecs_iso2022
+DEFINE_CODEC(iso2022_kr)
+DEFINE_CODEC(iso2022_jp)
+DEFINE_CODEC(iso2022_jp_1)
+DEFINE_CODEC(iso2022_jp_2)
+DEFINE_CODEC(iso2022_jp_2004)
+DEFINE_CODEC(iso2022_jp_3)
+DEFINE_CODEC(iso2022_jp_ext)
+
+//_codecs_jp
+DEFINE_CODEC(shift_jis)
+DEFINE_CODEC(cp932)
+DEFINE_CODEC(euc_jp)
+DEFINE_CODEC(shift_jis_2004)
+DEFINE_CODEC(euc_jis_2004)
+DEFINE_CODEC(euc_jisx0213)
+DEFINE_CODEC(shift_jisx0213)
+
+//_codecs_kr
+DEFINE_CODEC(euc_kr)
+DEFINE_CODEC(cp949)
+DEFINE_CODEC(johab)
+
+//_codecs_tw
+DEFINE_CODEC(big5)
+DEFINE_CODEC(cp950)
+
+
#endif
_______________________________________________
pypy-commit mailing list
[email protected]
http://mail.python.org/mailman/listinfo/pypy-commit
