Supplementary Plane characters can exist in UTF-16 files, but they are not valid UCS-2 characters.
For example, refer to this python interpreter code: >>> import codecs >>> codecs.encode(u'\U00010300', 'utf-16') '\xff\xfe\x00\xd8\x00\xdf' Therefore the UCS-4 0x00010300 character is encoded as two 16-bit numbers (0xd800 0xdf00) in a little endian UTF-16 file. For more information, see: http://en.wikipedia.org/wiki/UTF-16#U.2B10000_to_U.2B10FFFF This means that our current BaseTools code could be allowing unsupported UTF-16 characters be used. To fix this, we decode the file using python's utf-16 decode support. Then we verify that each character's code point is 0xffff or less. v3: * Based on Mike Kinney's feedback, we now read the whole file and verify up-front that it contains valid UCS-2 characters. Thanks also to Laszlo Ersek for pointing out the Supplementary Plane characters. v4: * Reject code points in 0xd800-0xdfff range since they are reserved for UTF-16 surrogate pairs. (lersek) Contributed-under: TianoCore Contribution Agreement 1.0 Signed-off-by: Jordan Justen <jordan.l.jus...@intel.com> Cc: Yingke D Liu <yingke.d....@intel.com> Cc: Michael D Kinney <michael.d.kin...@intel.com> Cc: Laszlo Ersek <ler...@redhat.com> --- BaseTools/Source/Python/AutoGen/UniClassObject.py | 88 ++++++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/UniClassObject.py b/BaseTools/Source/Python/AutoGen/UniClassObject.py index aa54f4f..386e1ec 100644 --- a/BaseTools/Source/Python/AutoGen/UniClassObject.py +++ b/BaseTools/Source/Python/AutoGen/UniClassObject.py @@ -19,6 +19,7 @@ import Common.LongFilePathOs as os, codecs, re import distutils.util import Common.EdkLogger as EdkLogger +import StringIO from Common.BuildToolError import * from Common.String import GetLineNo from Common.Misc import PathClass @@ -147,6 +148,37 @@ def GetLanguageCode(LangName, IsCompatibleMode, File): EdkLogger.error("Unicode File Parser", FORMAT_INVALID, "Invalid RFC 4646 language code : %s" % LangName, File) +## Ucs2Codec +# +# This is only a partial codec implementation. It only supports +# encoding, and is primarily used to check that all the characters are +# valid for UCS-2. +# +class Ucs2Codec(codecs.Codec): + def __init__(self): + self.__utf16 = codecs.lookup('utf-16') + + def encode(self, input, errors='strict'): + for Char in input: + CodePoint = ord(Char) + if CodePoint >= 0xd800 and CodePoint <= 0xdfff: + raise ValueError("Code Point is in range reserved for " + + "UTF-16 surrogate pairs") + elif CodePoint > 0xffff: + raise ValueError("Code Point too large to encode in UCS-2") + return self.__utf16.encode(input) + +TheUcs2Codec = Ucs2Codec() +def Ucs2Search(name): + if name == 'ucs-2': + return codecs.CodecInfo( + name=name, + encode=TheUcs2Codec.encode, + decode=TheUcs2Codec.decode) + else: + return None +codecs.register(Ucs2Search) + ## StringDefClassObject # # A structure for language definition @@ -209,7 +241,7 @@ class UniFileClassObject(object): Lang = distutils.util.split_quoted((Line.split(u"//")[0])) if len(Lang) != 3: try: - FileIn = codecs.open(LongFilePath(File.Path), mode='rb', encoding='utf-16').read() + FileIn = self.OpenUniFile(LongFilePath(File.Path)) except UnicodeError, X: EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File); except: @@ -253,6 +285,58 @@ class UniFileClassObject(object): self.OrderedStringDict[LangName][Item.StringName] = len(self.OrderedStringList[LangName]) - 1 return True + def OpenUniFile(self, FileName): + # + # Read file + # + try: + UniFile = open(FileName, mode='rb') + FileIn = UniFile.read() + UniFile.close() + except: + EdkLogger.Error("build", FILE_OPEN_FAILURE, ExtraData=File) + + # + # We currently only support UTF-16 + # + Encoding = 'utf-16' + + self.VerifyUcs2Data(FileIn, FileName, Encoding) + + UniFile = StringIO.StringIO(FileIn) + Info = codecs.lookup(Encoding) + (Reader, Writer) = (Info.streamreader, Info.streamwriter) + return codecs.StreamReaderWriter(UniFile, Reader, Writer) + + def VerifyUcs2Data(self, FileIn, FileName, Encoding): + Ucs2Info = codecs.lookup('ucs-2') + # + # Convert to unicode + # + try: + FileDecoded = codecs.decode(FileIn, Encoding) + Ucs2Info.encode(FileDecoded) + except: + UniFile = StringIO.StringIO(FileIn) + Info = codecs.lookup(Encoding) + (Reader, Writer) = (Info.streamreader, Info.streamwriter) + File = codecs.StreamReaderWriter(UniFile, Reader, Writer) + LineNumber = 0 + ErrMsg = lambda Encoding, LineNumber: \ + '%s contains invalid %s characters on line %d.' % \ + (FileName, Encoding, LineNumber) + while True: + LineNumber = LineNumber + 1 + try: + Line = File.readline() + if Line == '': + EdkLogger.error('Unicode File Parser', PARSER_ERROR, + ErrMsg(Encoding, LineNumber)) + Ucs2Info.encode(Line) + except: + EdkLogger.error('Unicode File Parser', PARSER_ERROR, + ErrMsg('UCS-2', LineNumber)) + # # Get String name and value # @@ -305,7 +389,7 @@ class UniFileClassObject(object): EdkLogger.error("Unicode File Parser", FILE_NOT_FOUND, ExtraData=File.Path) try: - FileIn = codecs.open(LongFilePath(File.Path), mode='rb', encoding='utf-16') + FileIn = self.OpenUniFile(LongFilePath(File.Path)) except UnicodeError, X: EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File.Path); except: -- 2.1.4 ------------------------------------------------------------------------------ _______________________________________________ edk2-devel mailing list edk2-devel@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/edk2-devel
