MarcusJohnson91 updated this revision to Diff 361487.
MarcusJohnson91 edited the summary of this revision.
MarcusJohnson91 added a comment.
Added tests
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D106753/new/
https://reviews.llvm.org/D106753
Files:
llvm/include/llvm/Support/ConvertUTF.h
llvm/lib/Support/ConvertUTFWrapper.cpp
llvm/unittests/Support/ConvertUTFTest.cpp
Index: llvm/unittests/Support/ConvertUTFTest.cpp
===================================================================
--- llvm/unittests/Support/ConvertUTFTest.cpp
+++ llvm/unittests/Support/ConvertUTFTest.cpp
@@ -36,6 +36,28 @@
EXPECT_EQ(Expected, Result);
}
+TEST(ConvertUTFTest, ConvertUTF32LittleEndianToUTF8String) {
+ // Src is the look of disapproval.
+ alignas(UTF32) static const char Src[] = "\xff\xfe\xa0\x0c_\x00\xa0\x0c";
+ ArrayRef<char> Ref(Src, sizeof(Src) - 1);
+ std::string Result;
+ bool Success = convertUTF32ToUTF8String(Ref, Result);
+ EXPECT_TRUE(Success);
+ std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0");
+ EXPECT_EQ(Expected, Result);
+}
+
+TEST(ConvertUTFTest, ConvertUTF32BigEndianToUTF8String) {
+ // Src is the look of disapproval.
+ alignas(UTF32) static const char Src[] = "\xfe\xff\x0c\xa0\x00_\x0c\xa0";
+ ArrayRef<char> Ref(Src, sizeof(Src) - 1);
+ std::string Result;
+ bool Success = convertUTF32ToUTF8String(Ref, Result);
+ EXPECT_TRUE(Success);
+ std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0");
+ EXPECT_EQ(Expected, Result);
+}
+
TEST(ConvertUTFTest, ConvertUTF8ToUTF16String) {
// Src is the look of disapproval.
static const char Src[] = "\xe0\xb2\xa0_\xe0\xb2\xa0";
@@ -78,6 +100,33 @@
EXPECT_FALSE(HasBOM);
}
+TEST(ConvertUTFTest, HasUTF32BOM) {
+ bool HasBOM = hasUTF32ByteOrderMark(makeArrayRef("\x00\x00\xfe\xff", 4));
+ EXPECT_TRUE(HasBOM);
+ HasBOM = hasUTF32ByteOrderMark(makeArrayRef("\xff\xfe\x00\x00", 4));
+ EXPECT_TRUE(HasBOM);
+ HasBOM = hasUTF32ByteOrderMark(makeArrayRef("\x00\x00\xfe\xff ", 5));
+ EXPECT_TRUE(HasBOM); // Don't care about odd lengths.
+ HasBOM = hasUTF32ByteOrderMark(makeArrayRef("\x00\x00\xfe\xff\x00asdf", 9));
+ EXPECT_TRUE(HasBOM);
+
+ HasBOM = hasUTF32ByteOrderMark(None);
+ EXPECT_FALSE(HasBOM);
+ HasBOM = hasUTF32ByteOrderMark(makeArrayRef("\xfe", 1));
+ EXPECT_FALSE(HasBOM);
+}
+
+TEST(ConvertUTFTest, UTF32WrappersForConvertUTF32ToUTF8String) {
+ // Src is the look of disapproval.
+ alignas(UTF32) static const char Src[] = "\xff\xfe\xa0\x0c_\x00\xa0\x0c";
+ ArrayRef<UTF32> SrcRef = makeArrayRef((const UTF32 *)Src, 4);
+ std::string Result;
+ bool Success = convertUTF32ToUTF8String(SrcRef, Result);
+ EXPECT_TRUE(Success);
+ std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0");
+ EXPECT_EQ(Expected, Result);
+}
+
TEST(ConvertUTFTest, UTF16WrappersForConvertUTF16ToUTF8String) {
// Src is the look of disapproval.
alignas(UTF16) static const char Src[] = "\xff\xfe\xa0\x0c_\x00\xa0\x0c";
Index: llvm/lib/Support/ConvertUTFWrapper.cpp
===================================================================
--- llvm/lib/Support/ConvertUTFWrapper.cpp
+++ llvm/lib/Support/ConvertUTFWrapper.cpp
@@ -82,6 +82,12 @@
((S[0] == '\xff' && S[1] == '\xfe') ||
(S[0] == '\xfe' && S[1] == '\xff')));
}
+
+bool hasUTF32ByteOrderMark(ArrayRef<char> S) {
+ return (S.size() >= 4 &&
+ ((S[0] == '\x00' && S[1] == '\x00' && S[2] == '\xfe' && S[3] == '\xff') ||
+ (S[0] == '\xff' && S[1] == '\xfe' && S[2] == '\x00' && S[3] == '\x00')));
+}
bool convertUTF16ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out) {
assert(Out.empty());
@@ -140,6 +146,64 @@
llvm::ArrayRef<char>(reinterpret_cast<const char *>(Src.data()),
Src.size() * sizeof(UTF16)), Out);
}
+
+bool convertUTF32ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out) {
+ assert(Out.empty());
+
+ // Error out on an uneven byte count.
+ if (SrcBytes.size() % 2)
+ return false;
+
+ // Avoid OOB by returning early on empty input.
+ if (SrcBytes.empty())
+ return true;
+
+ const UTF32 *Src = reinterpret_cast<const UTF32 *>(SrcBytes.begin());
+ const UTF32 *SrcEnd = reinterpret_cast<const UTF32 *>(SrcBytes.end());
+
+ assert((uintptr_t)Src % sizeof(UTF32) == 0);
+
+ // Byteswap if necessary.
+ std::vector<UTF32> ByteSwapped;
+ if (Src[0] == UNI_UTF16_BYTE_ORDER_MARK_SWAPPED) {
+ ByteSwapped.insert(ByteSwapped.end(), Src, SrcEnd);
+ for (unsigned I = 0, E = ByteSwapped.size(); I != E; ++I)
+ ByteSwapped[I] = llvm::ByteSwap_32(ByteSwapped[I]);
+ Src = &ByteSwapped[0];
+ SrcEnd = &ByteSwapped[ByteSwapped.size() - 1] + 1;
+ }
+
+ // Skip the BOM for conversion.
+ if (Src[0] == UNI_UTF32_BYTE_ORDER_MARK_NATIVE)
+ Src++;
+
+ // Just allocate enough space up front. We'll shrink it later. Allocate
+ // enough that we can fit a null terminator without reallocating.
+ Out.resize(SrcBytes.size() * UNI_MAX_UTF8_BYTES_PER_CODE_POINT + 1);
+ UTF8 *Dst = reinterpret_cast<UTF8 *>(&Out[0]);
+ UTF8 *DstEnd = Dst + Out.size();
+
+ ConversionResult CR =
+ ConvertUTF32toUTF8(&Src, SrcEnd, &Dst, DstEnd, strictConversion);
+ assert(CR != targetExhausted);
+
+ if (CR != conversionOK) {
+ Out.clear();
+ return false;
+ }
+
+ Out.resize(reinterpret_cast<char *>(Dst) - &Out[0]);
+ Out.push_back(0);
+ Out.pop_back();
+ return true;
+}
+
+bool convertUTF32ToUTF8String(ArrayRef<UTF32> Src, std::string &Out)
+{
+ return convertUTF16ToUTF8String(
+ llvm::ArrayRef<char>(reinterpret_cast<const char *>(Src.data()),
+ Src.size() * sizeof(UTF32)), Out);
+}
bool convertUTF8ToUTF16String(StringRef SrcUTF8,
SmallVectorImpl<UTF16> &DstUTF16) {
Index: llvm/include/llvm/Support/ConvertUTF.h
===================================================================
--- llvm/include/llvm/Support/ConvertUTF.h
+++ llvm/include/llvm/Support/ConvertUTF.h
@@ -122,6 +122,9 @@
#define UNI_UTF16_BYTE_ORDER_MARK_NATIVE 0xFEFF
#define UNI_UTF16_BYTE_ORDER_MARK_SWAPPED 0xFFFE
+
+#define UNI_UTF32_BYTE_ORDER_MARK_NATIVE 0x0000FEFF
+#define UNI_UTF32_BYTE_ORDER_MARK_SWAPPED 0x0000FFFE
typedef enum {
conversionOK, /* conversion successful */
@@ -259,6 +262,12 @@
* order mark.
*/
bool hasUTF16ByteOrderMark(ArrayRef<char> SrcBytes);
+
+/**
+ * Returns true if a blob of text starts with a UTF-32 big or little endian byte
+ * order mark.
+ */
+bool hasUTF32ByteOrderMark(ArrayRef<char> SrcBytes);
/**
* Converts a stream of raw bytes assumed to be UTF16 into a UTF8 std::string.
@@ -277,6 +286,24 @@
* \returns true on success
*/
bool convertUTF16ToUTF8String(ArrayRef<UTF16> Src, std::string &Out);
+
+/**
+ * Converts a stream of raw bytes assumed to be UTF32 into a UTF8 std::string.
+ *
+ * \param [in] SrcBytes A buffer of what is assumed to be UTF-32 encoded text.
+ * \param [out] Out Converted UTF-8 is stored here on success.
+ * \returns true on success
+ */
+bool convertUTF32ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out);
+
+/**
+* Converts a UTF32 string into a UTF8 std::string.
+*
+* \param [in] Src A buffer of UTF-32 encoded text.
+* \param [out] Out Converted UTF-8 is stored here on success.
+* \returns true on success
+*/
+bool convertUTF32ToUTF8String(ArrayRef<UTF32> Src, std::string &Out);
/**
* Converts a UTF-8 string into a UTF-16 string with native endianness.
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
