Hi djasper,
Count column width instead of the number of code points. This also
includes correct handling of tabs inside string literals and comments (with an
exception of multiline string literals/comments, where tabs are present before
the first escaped newline).
http://llvm-reviews.chandlerc.com/D1601
Files:
lib/Format/BreakableToken.cpp
lib/Format/ContinuationIndenter.cpp
lib/Format/Encoding.h
lib/Format/Format.cpp
lib/Format/FormatToken.h
unittests/Format/FormatTest.cpp
Index: lib/Format/BreakableToken.cpp
===================================================================
--- lib/Format/BreakableToken.cpp
+++ lib/Format/BreakableToken.cpp
@@ -49,9 +49,13 @@
unsigned MaxSplitBytes = 0;
for (unsigned NumChars = 0;
- NumChars < MaxSplit && MaxSplitBytes < Text.size(); ++NumChars)
- MaxSplitBytes +=
+ NumChars < MaxSplit && MaxSplitBytes < Text.size();) {
+ unsigned NumBytes =
encoding::getCodePointNumBytes(Text[MaxSplitBytes], Encoding);
+ NumChars += encoding::columnWidthWithTabs(
+ Text.substr(MaxSplitBytes, NumBytes), ContentStartColumn, Encoding);
+ MaxSplitBytes += NumBytes;
+ }
StringRef::size_type SpaceOffset = Text.find_last_of(Blanks, MaxSplitBytes);
if (SpaceOffset == StringRef::npos ||
@@ -84,9 +88,9 @@
return BreakableToken::Split(StringRef::npos, 0);
if (ColumnLimit <= ContentStartColumn)
return BreakableToken::Split(StringRef::npos, 0);
- unsigned MaxSplit =
- std::min<unsigned>(ColumnLimit - ContentStartColumn,
- encoding::getCodePointCount(Text, Encoding) - 1);
+ unsigned MaxSplit = std::min<unsigned>(
+ ColumnLimit - ContentStartColumn,
+ encoding::columnWidthWithTabs(Text, ContentStartColumn, Encoding) - 1);
StringRef::size_type SpaceOffset = 0;
StringRef::size_type SlashOffset = 0;
StringRef::size_type WordStartOffset = 0;
@@ -98,7 +102,8 @@
Chars += Advance;
} else {
Advance = encoding::getCodePointNumBytes(Text[0], Encoding);
- Chars += 1;
+ Chars += encoding::columnWidthWithTabs(
+ Text.substr(0, Advance), ContentStartColumn + Chars, Encoding);
}
if (Chars > MaxSplit)
@@ -131,7 +136,8 @@
unsigned BreakableSingleLineToken::getLineLengthAfterSplit(
unsigned LineIndex, unsigned Offset, StringRef::size_type Length) const {
return StartColumn + Prefix.size() + Postfix.size() +
- encoding::getCodePointCount(Line.substr(Offset, Length), Encoding);
+ encoding::columnWidthWithTabs(Line.substr(Offset, Length),
+ StartColumn + Prefix.size(), Encoding);
}
BreakableSingleLineToken::BreakableSingleLineToken(
@@ -328,9 +334,10 @@
unsigned BreakableBlockComment::getLineLengthAfterSplit(
unsigned LineIndex, unsigned Offset, StringRef::size_type Length) const {
- return getContentStartColumn(LineIndex, Offset) +
- encoding::getCodePointCount(Lines[LineIndex].substr(Offset, Length),
- Encoding) +
+ unsigned ContentStartColumn = getContentStartColumn(LineIndex, Offset);
+ return ContentStartColumn +
+ encoding::columnWidthWithTabs(Lines[LineIndex].substr(Offset, Length),
+ ContentStartColumn, Encoding) +
// The last line gets a "*/" postfix.
(LineIndex + 1 == Lines.size() ? 2 : 0);
}
Index: lib/Format/ContinuationIndenter.cpp
===================================================================
--- lib/Format/ContinuationIndenter.cpp
+++ lib/Format/ContinuationIndenter.cpp
@@ -588,10 +588,10 @@
State.Stack[i].BreakBeforeParameter = true;
unsigned ColumnsUsed =
- State.Column - Current.CodePointCount + Current.CodePointsInFirstLine;
+ State.Column - Current.CodePointCount + Current.FirstLineColumnWidth;
// We can only affect layout of the first and the last line, so the penalty
// for all other lines is constant, and we ignore it.
- State.Column = Current.CodePointsInLastLine;
+ State.Column = Current.LastLineColumnWidth;
if (ColumnsUsed > getColumnLimit())
return Style.PenaltyExcessCharacter * (ColumnsUsed - getColumnLimit());
@@ -643,7 +643,7 @@
// leading whitespace in consecutive lines when changing indentation of
// the first line similar to what we do with block comments.
if (Current.isMultiline()) {
- State.Column = StartColumn + Current.CodePointsInFirstLine;
+ State.Column = StartColumn + Current.FirstLineColumnWidth;
return 0;
}
Index: lib/Format/Encoding.h
===================================================================
--- lib/Format/Encoding.h
+++ lib/Format/Encoding.h
@@ -18,6 +18,7 @@
#include "clang/Basic/LLVM.h"
#include "llvm/Support/ConvertUTF.h"
+#include "llvm/Support/Unicode.h"
namespace clang {
namespace format {
@@ -57,6 +58,31 @@
}
}
+inline int columnWidth(StringRef Text, Encoding Encoding) {
+ if (Encoding == Encoding_UTF8) {
+ int ContentWidth = llvm::sys::unicode::columnWidthUTF8(Text);
+ if (ContentWidth >= 0)
+ return ContentWidth;
+ }
+ return getCodePointCount(Text, Encoding);
+}
+
+// StartColumn is used for tab character handling.
+inline unsigned columnWidthWithTabs(StringRef Text, int StartColumn,
+ Encoding Encoding) {
+ unsigned TotalWidth = 0;
+ for (;;) {
+ StringRef::size_type TabPos = Text.find('\t');
+ if (TabPos == StringRef::npos)
+ return TotalWidth + columnWidth(Text, Encoding);
+ int Width = columnWidth(Text.substr(0, TabPos), Encoding);
+ assert(Width >= 0);
+ TotalWidth += Width;
+ TotalWidth += 8 - (TotalWidth + StartColumn) % 8;
+ Text = Text.substr(TabPos + 1);
+ }
+}
+
/// \brief Gets the number of bytes in a sequence representing a single
/// codepoint and starting with FirstChar in the specified Encoding.
inline unsigned getCodePointNumBytes(char FirstChar, Encoding Encoding) {
Index: lib/Format/Format.cpp
===================================================================
--- lib/Format/Format.cpp
+++ lib/Format/Format.cpp
@@ -603,10 +603,11 @@
StringRef Text = FormatTok->TokenText;
size_t FirstNewlinePos = Text.find('\n');
if (FirstNewlinePos != StringRef::npos) {
- FormatTok->CodePointsInFirstLine = encoding::getCodePointCount(
- Text.substr(0, FirstNewlinePos), Encoding);
- FormatTok->CodePointsInLastLine = encoding::getCodePointCount(
- Text.substr(Text.find_last_of('\n') + 1), Encoding);
+ // FIXME: Handle embedded tabs.
+ FormatTok->FirstLineColumnWidth = encoding::columnWidthWithTabs(
+ Text.substr(0, FirstNewlinePos), 0, Encoding);
+ FormatTok->LastLineColumnWidth = encoding::columnWidthWithTabs(
+ Text.substr(Text.find_last_of('\n') + 1), 0, Encoding);
}
}
// FIXME: Add the CodePointCount to Column.
Index: lib/Format/FormatToken.h
===================================================================
--- lib/Format/FormatToken.h
+++ lib/Format/FormatToken.h
@@ -81,7 +81,7 @@
struct FormatToken {
FormatToken()
: NewlinesBefore(0), HasUnescapedNewline(false), LastNewlineOffset(0),
- CodePointCount(0), CodePointsInFirstLine(0), CodePointsInLastLine(0),
+ CodePointCount(0), FirstLineColumnWidth(0), LastLineColumnWidth(0),
IsFirst(false), MustBreakBefore(false), IsUnterminatedLiteral(false),
BlockKind(BK_Unknown), Type(TT_Unknown), SpacesRequiredBefore(0),
CanBreakBefore(false), ClosesTemplateDeclaration(false),
@@ -118,15 +118,15 @@
/// \brief Contains the number of code points in the first line of a
/// multi-line string literal or comment. Zero if there's no newline in the
/// token.
- unsigned CodePointsInFirstLine;
+ unsigned FirstLineColumnWidth;
/// \brief Contains the number of code points in the last line of a
/// multi-line string literal or comment. Can be zero for line comments.
- unsigned CodePointsInLastLine;
+ unsigned LastLineColumnWidth;
/// \brief Returns \c true if the token text contains newlines (escaped or
/// not).
- bool isMultiline() const { return CodePointsInFirstLine != 0; }
+ bool isMultiline() const { return FirstLineColumnWidth != 0; }
/// \brief Indicates that this is the first token.
bool IsFirst;
Index: unittests/Format/FormatTest.cpp
===================================================================
--- unittests/Format/FormatTest.cpp
+++ unittests/Format/FormatTest.cpp
@@ -6058,15 +6058,15 @@
verifyFormat("\"Однажды в студёную зимнюю пору...\"",
getLLVMStyleWithColumns(35));
verifyFormat("\"一 二 三 四 五 六 七 八 九 十\"",
- getLLVMStyleWithColumns(21));
+ getLLVMStyleWithColumns(31));
verifyFormat("// Однажды в студёную зимнюю пору...",
getLLVMStyleWithColumns(36));
verifyFormat("// 一 二 三 四 五 六 七 八 九 十",
- getLLVMStyleWithColumns(22));
+ getLLVMStyleWithColumns(32));
verifyFormat("/* Однажды в студёную зимнюю пору... */",
getLLVMStyleWithColumns(39));
verifyFormat("/* 一 二 三 四 五 六 七 八 九 十 */",
- getLLVMStyleWithColumns(25));
+ getLLVMStyleWithColumns(35));
}
TEST_F(FormatTest, SplitsUTF8Strings) {
@@ -6077,11 +6077,29 @@
"\"пору,\"",
format("\"Однажды, в студёную зимнюю пору,\"",
getLLVMStyleWithColumns(13)));
- EXPECT_EQ("\"一 二 三 四 \"\n"
- "\"五 六 七 八 \"\n"
- "\"九 十\"",
- format("\"一 二 三 四 五 六 七 八 九 十\"",
- getLLVMStyleWithColumns(10)));
+ EXPECT_EQ("\"一 二 三 \"\n"
+ "\"四 五六 \"\n"
+ "\"七 八 九 \"\n"
+ "\"十\"",
+ format("\"一 二 三 四 五六 七 八 九 十\"",
+ getLLVMStyleWithColumns(11)));
+ EXPECT_EQ("\"一\t二 \"\n"
+ "\"\t三 \"\n"
+ "\"四 五\t六 \"\n"
+ "\"\t七 \"\n"
+ "\"八九十\tqq\"",
+ format("\"一\t二 \t三 四 五\t六 \t七 八九十\tqq\"",
+ getLLVMStyleWithColumns(11)));
+}
+
+
+TEST_F(FormatTest, HandlesDoubleWidthCharsInMultiLineStrings) {
+ EXPECT_EQ("const char *sssss =\n"
+ " \"一二三四五六七八\\\n"
+ " 九 十\";",
+ format("const char *sssss = \"一二三四五六七八\\\n"
+ " 九 十\";",
+ getLLVMStyleWithColumns(30)));
}
TEST_F(FormatTest, SplitsUTF8LineComments) {
@@ -6093,9 +6111,9 @@
getLLVMStyleWithColumns(13)));
EXPECT_EQ("// 一二三\n"
"// 四五六七\n"
- "// 八\n"
- "// 九 十",
- format("// 一二三 四五六七 八 九 十", getLLVMStyleWithColumns(6)));
+ "// 八 九\n"
+ "// 十",
+ format("// 一二三 四五六七 八 九 十", getLLVMStyleWithColumns(9)));
}
TEST_F(FormatTest, SplitsUTF8BlockComments) {
@@ -6110,10 +6128,10 @@
format("/* Гляжу, поднимается медленно в гору\n"
" * Лошадка, везущая хворосту воз. */",
getLLVMStyleWithColumns(13)));
- EXPECT_EQ("/* 一二三\n"
- " * 四五六七\n"
- " * 八\n"
- " * 九 十\n"
- " */",
- format("/* 一二三 四五六七 八 九 十 */", getLLVMStyleWithColumns(6)));
+ EXPECT_EQ(
+ "/* 一二三\n"
+ " * 四五六七\n"
+ " * 八 九\n"
+ " * 十 */",
+ format("/* 一二三 四五六七 八 九 十 */", getLLVMStyleWithColumns(9)));
EXPECT_EQ("/*
_______________________________________________
cfe-commits mailing list
[email protected]
http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits
