This is an automated email from the ASF dual-hosted git repository. joshtynjala pushed a commit to branch develop in repository https://gitbox.apache.org/repos/asf/royale-compiler.git
The following commit(s) were added to refs/heads/develop by this push: new fce284723 ASTokenFormatter: fix unicode and ascii escape sequences getting lost when formatting string literals fce284723 is described below commit fce284723bcbe04443901a8f4c006125c31f3947 Author: Josh Tynjala <joshtynj...@apache.org> AuthorDate: Wed Sep 13 09:45:31 2023 -0700 ASTokenFormatter: fix unicode and ascii escape sequences getting lost when formatting string literals Now copies the string from the original source code as-sis instead of trying to convert back to escaped characters --- .../apache/royale/formatter/ASTokenFormatter.java | 32 +++-- .../apache/royale/formatter/TestStringLiteral.java | 142 ++++++++++++++++++++- 2 files changed, 163 insertions(+), 11 deletions(-) diff --git a/formatter/src/main/java/org/apache/royale/formatter/ASTokenFormatter.java b/formatter/src/main/java/org/apache/royale/formatter/ASTokenFormatter.java index 27c5ca2ca..3cddb3214 100644 --- a/formatter/src/main/java/org/apache/royale/formatter/ASTokenFormatter.java +++ b/formatter/src/main/java/org/apache/royale/formatter/ASTokenFormatter.java @@ -170,7 +170,7 @@ public class ASTokenFormatter extends BaseTokenFormatter { List<IASToken> tokens = insertExtraAS3Tokens(repairedTokensList, text); try { - return parseTokens(tokens); + return parseTokens(tokens, text); } catch (Exception e) { if (problems != null) { System.err.println(e); @@ -181,7 +181,7 @@ public class ASTokenFormatter extends BaseTokenFormatter { } } - private String parseTokens(List<IASToken> tokens) throws Exception { + private String parseTokens(List<IASToken> tokens, String fileText) throws Exception { indent = 0; inCaseOrDefaultClause = false; inControlFlowStatement = false; @@ -513,7 +513,7 @@ public class ASTokenFormatter extends BaseTokenFormatter { } // include the token's own text - builder.append(getTokenText(token, indent, skipFormatting)); + builder.append(getTokenText(token, indent, skipFormatting, fileText)); // characters that must appear after the token if (token.getType() != ASTokenTypes.HIDDEN_TOKEN_SINGLE_LINE_COMMENT @@ -1109,7 +1109,7 @@ public class ASTokenFormatter extends BaseTokenFormatter { } } - private String getTokenText(IASToken token, int indent, boolean skipFormatting) { + private String getTokenText(IASToken token, int indent, boolean skipFormatting, String fileText) { if (token instanceof MetaDataPayloadToken) { MetaDataPayloadToken metaPlayloadToken = (MetaDataPayloadToken) token; return formatMetadataToken(metaPlayloadToken); @@ -1134,7 +1134,7 @@ public class ASTokenFormatter extends BaseTokenFormatter { return formatMultiLineComment(token.getText()); } case ASTokenTypes.TOKEN_LITERAL_STRING: { - return formatLiteralString(token); + return formatLiteralString(token, fileText); } case ASTokenTypes.TOKEN_SEMICOLON: { if (skipFormatting) { @@ -1233,7 +1233,19 @@ public class ASTokenFormatter extends BaseTokenFormatter { return comment; } - private String formatLiteralString(IASToken token) { + private String formatLiteralString(IASToken token, String fileText) { + int start = token.getAbsoluteStart(); + int end = token.getAbsoluteEnd(); + if (start != -1 && start < end && end < fileText.length()) { + // escape sequences are converted to real characters when the + // original source code is converted to to tokens + // the user won't be happy if their strings get changed + // (and, in some cases, it may become an invalid string), + // so grab the original string from the file + return fileText.substring(start, end); + } + // we should never get here, but this should handle most strings, if + // something unexpected happens String string = token.getText(); String charsToEscape = "\b\t\n\f\r\\"; String escapeChars = "btnfr\\"; @@ -1395,7 +1407,7 @@ public class ASTokenFormatter extends BaseTokenFormatter { return numNewLinesInWhitespace; } - private List<IASToken> insertExtraAS3Tokens(List<IASToken> originalTokens, String text) { + private List<IASToken> insertExtraAS3Tokens(List<IASToken> originalTokens, String fileText) { ArrayList<IASToken> tokens = new ArrayList<IASToken>(); IASToken prevToken = null; for (IASToken token : originalTokens) { @@ -1411,7 +1423,7 @@ public class ASTokenFormatter extends BaseTokenFormatter { int start = prevToken.getAbsoluteEnd(); int end = token.getAbsoluteStart(); if (end > start) { - String tokenText = text.substring(start, end); + String tokenText = fileText.substring(start, end); ASToken extraToken = new ASToken(TOKEN_TYPE_EXTRA, start, end, prevToken.getEndLine(), prevToken.getEndColumn(), tokenText); extraToken.setEndLine(token.getLine()); @@ -1424,9 +1436,9 @@ public class ASTokenFormatter extends BaseTokenFormatter { } if (prevToken != null) { int start = prevToken.getAbsoluteEnd(); - int end = text.length(); + int end = fileText.length(); if (end > start) { - String tokenText = text.substring(start, end); + String tokenText = fileText.substring(start, end); ASToken extraToken = new ASToken(TOKEN_TYPE_EXTRA, start, end, prevToken.getEndLine(), prevToken.getEndColumn(), tokenText); extraToken.setEndLine(prevToken.getLine()); diff --git a/formatter/src/test/java/org/apache/royale/formatter/TestStringLiteral.java b/formatter/src/test/java/org/apache/royale/formatter/TestStringLiteral.java index 90b64b65c..19235bf28 100644 --- a/formatter/src/test/java/org/apache/royale/formatter/TestStringLiteral.java +++ b/formatter/src/test/java/org/apache/royale/formatter/TestStringLiteral.java @@ -45,7 +45,47 @@ public class TestStringLiteral extends BaseFormatterTests { } @Test - public void testWithNewLine() { + public void testTabEscape() { + FormatterSettings settings = new FormatterSettings(); + settings.insertSpaceAfterKeywordsInControlFlowStatements = true; + settings.placeOpenBraceOnNewLine = true; + settings.insertSpaces = false; + ASTokenFormatter formatter = new ASTokenFormatter(settings); + String result = formatter.format("file.as", + // @formatter:off + "\"\\t\";", + // @formatter:on + problems + ); + assertEquals( + // @formatter:off + "\"\\t\";", + // @formatter:on + result); + } + + @Test + public void testCarriageReturnEscape() { + FormatterSettings settings = new FormatterSettings(); + settings.insertSpaceAfterKeywordsInControlFlowStatements = true; + settings.placeOpenBraceOnNewLine = true; + settings.insertSpaces = false; + ASTokenFormatter formatter = new ASTokenFormatter(settings); + String result = formatter.format("file.as", + // @formatter:off + "\"\\r\";", + // @formatter:on + problems + ); + assertEquals( + // @formatter:off + "\"\\r\";", + // @formatter:on + result); + } + + @Test + public void testNewLineEscape() { FormatterSettings settings = new FormatterSettings(); settings.insertSpaceAfterKeywordsInControlFlowStatements = true; settings.placeOpenBraceOnNewLine = true; @@ -64,6 +104,106 @@ public class TestStringLiteral extends BaseFormatterTests { result); } + @Test + public void testBackslashEscape() { + FormatterSettings settings = new FormatterSettings(); + settings.insertSpaceAfterKeywordsInControlFlowStatements = true; + settings.placeOpenBraceOnNewLine = true; + settings.insertSpaces = false; + ASTokenFormatter formatter = new ASTokenFormatter(settings); + String result = formatter.format("file.as", + // @formatter:off + "\"\\\\\";", + // @formatter:on + problems + ); + assertEquals( + // @formatter:off + "\"\\\\\";", + // @formatter:on + result); + } + + @Test + public void testBackspaceEscape() { + FormatterSettings settings = new FormatterSettings(); + settings.insertSpaceAfterKeywordsInControlFlowStatements = true; + settings.placeOpenBraceOnNewLine = true; + settings.insertSpaces = false; + ASTokenFormatter formatter = new ASTokenFormatter(settings); + String result = formatter.format("file.as", + // @formatter:off + "\"\\b\";", + // @formatter:on + problems + ); + assertEquals( + // @formatter:off + "\"\\b\";", + // @formatter:on + result); + } + + @Test + public void testFormFeedEscape() { + FormatterSettings settings = new FormatterSettings(); + settings.insertSpaceAfterKeywordsInControlFlowStatements = true; + settings.placeOpenBraceOnNewLine = true; + settings.insertSpaces = false; + ASTokenFormatter formatter = new ASTokenFormatter(settings); + String result = formatter.format("file.as", + // @formatter:off + "\"\\f\";", + // @formatter:on + problems + ); + assertEquals( + // @formatter:off + "\"\\f\";", + // @formatter:on + result); + } + + @Test + public void testAsciiEscape() { + FormatterSettings settings = new FormatterSettings(); + settings.insertSpaceAfterKeywordsInControlFlowStatements = true; + settings.placeOpenBraceOnNewLine = true; + settings.insertSpaces = false; + ASTokenFormatter formatter = new ASTokenFormatter(settings); + String result = formatter.format("file.as", + // @formatter:off + "\"\\x21\";", + // @formatter:on + problems + ); + assertEquals( + // @formatter:off + "\"\\x21\";", + // @formatter:on + result); + } + + @Test + public void testUnicodeEscape() { + FormatterSettings settings = new FormatterSettings(); + settings.insertSpaceAfterKeywordsInControlFlowStatements = true; + settings.placeOpenBraceOnNewLine = true; + settings.insertSpaces = false; + ASTokenFormatter formatter = new ASTokenFormatter(settings); + String result = formatter.format("file.as", + // @formatter:off + "\"\\u263a\";", + // @formatter:on + problems + ); + assertEquals( + // @formatter:off + "\"\\u263a\";", + // @formatter:on + result); + } + @Test public void testDoubleQuoteWithEscapedDoubleQuote() { FormatterSettings settings = new FormatterSettings();