This is an automated email from the ASF dual-hosted git repository. ddekany pushed a commit to branch 2.3-gae in repository https://gitbox.apache.org/repos/asf/freemarker.git
commit 11c2b09574899f1779ad4941adc1994455d53828 Author: ddekany <[email protected]> AuthorDate: Tue Oct 13 17:13:39 2020 +0200 Added ?eval_json to evaluate JSON given as flat string. This was added as ?eval is routinely misused for the same purpose. --- src/main/java/freemarker/core/BuiltIn.java | 4 +- .../freemarker/core/BuiltInsForStringsMisc.java | 17 +- src/main/java/freemarker/core/JSONParser.java | 622 +++++++++++++++++++++ src/manual/en_US/book.xml | 98 ++++ .../java/freemarker/core/EvalJsonBuiltInTest.java | 41 ++ src/test/java/freemarker/core/JSONParserTest.java | 171 ++++++ 6 files changed, 951 insertions(+), 2 deletions(-) diff --git a/src/main/java/freemarker/core/BuiltIn.java b/src/main/java/freemarker/core/BuiltIn.java index 30be937..fcea193 100644 --- a/src/main/java/freemarker/core/BuiltIn.java +++ b/src/main/java/freemarker/core/BuiltIn.java @@ -65,6 +65,7 @@ import freemarker.core.BuiltInsForSequences.sequenceBI; import freemarker.core.BuiltInsForSequences.sortBI; import freemarker.core.BuiltInsForSequences.sort_byBI; import freemarker.core.BuiltInsForStringsMisc.evalBI; +import freemarker.core.BuiltInsForStringsMisc.evalJsonBI; import freemarker.template.Configuration; import freemarker.template.TemplateDateModel; import freemarker.template.TemplateModel; @@ -84,7 +85,7 @@ abstract class BuiltIn extends Expression implements Cloneable { static final Set<String> CAMEL_CASE_NAMES = new TreeSet<>(); static final Set<String> SNAKE_CASE_NAMES = new TreeSet<>(); - static final int NUMBER_OF_BIS = 289; + static final int NUMBER_OF_BIS = 291; static final HashMap<String, BuiltIn> BUILT_INS_BY_NAME = new HashMap(NUMBER_OF_BIS * 3 / 2 + 1, 1f); static final String BI_NAME_SNAKE_CASE_WITH_ARGS = "with_args"; @@ -120,6 +121,7 @@ abstract class BuiltIn extends Expression implements Cloneable { putBI("ensure_starts_with", "ensureStartsWith", new BuiltInsForStringsBasic.ensure_starts_withBI()); putBI("esc", new escBI()); putBI("eval", new evalBI()); + putBI("eval_json", "evalJson", new evalJsonBI()); putBI("exists", new BuiltInsForExistenceHandling.existsBI()); putBI("filter", new BuiltInsForSequences.filterBI()); putBI("first", new firstBI()); diff --git a/src/main/java/freemarker/core/BuiltInsForStringsMisc.java b/src/main/java/freemarker/core/BuiltInsForStringsMisc.java index 012a007..284abee 100644 --- a/src/main/java/freemarker/core/BuiltInsForStringsMisc.java +++ b/src/main/java/freemarker/core/BuiltInsForStringsMisc.java @@ -113,6 +113,22 @@ class BuiltInsForStringsMisc { } + static class evalJsonBI extends BuiltInForString { + @Override + TemplateModel calculateResult(String s, Environment env) throws TemplateException { + try { + return JSONParser.parse(s); + } catch (JSONParser.JSONParseException e) { + throw new _MiscTemplateException(this, env, + "Failed to \"?", key, "\" string with this error:\n\n", + _MessageUtil.EMBEDDED_MESSAGE_BEGIN, + new _DelayedGetMessage(e), + _MessageUtil.EMBEDDED_MESSAGE_END, + "\n\nThe failing expression:"); + } + } + } + static class numberBI extends BuiltInForString { @Override TemplateModel calculateResult(String s, Environment env) throws TemplateException { @@ -170,5 +186,4 @@ class BuiltInsForStringsMisc { // Can't be instantiated private BuiltInsForStringsMisc() { } - } diff --git a/src/main/java/freemarker/core/JSONParser.java b/src/main/java/freemarker/core/JSONParser.java new file mode 100644 index 0000000..ddb01c0 --- /dev/null +++ b/src/main/java/freemarker/core/JSONParser.java @@ -0,0 +1,622 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package freemarker.core; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import freemarker.template.SimpleHash; +import freemarker.template.SimpleNumber; +import freemarker.template.SimpleScalar; +import freemarker.template.SimpleSequence; +import freemarker.template.Template; +import freemarker.template.TemplateBooleanModel; +import freemarker.template.TemplateHashModelEx2; +import freemarker.template.TemplateModel; +import freemarker.template.TemplateModelException; +import freemarker.template.TemplateNumberModel; +import freemarker.template.TemplateScalarModel; +import freemarker.template.TemplateSequenceModel; +import freemarker.template._TemplateAPI; +import freemarker.template.utility.Constants; +import freemarker.template.utility.NumberUtil; +import freemarker.template.utility.StringUtil; + +/** + * JSON parser that returns a {@link TameplatModel}, similar to what FTL literals product (and so, what + * @code ?eval} would return). A notable difference compared to the result FTL literals is that this doesn't use the + * {@link ParserConfiguration#getArithmeticEngine()} to parse numbers, as JSON has its own fixed number syntax. For + * numbers this parser returns {@link SimpleNumberModel}-s, where the wrapped numbers will be {@link Integer}-s when + * they fit into that, otherwise they will be {@link Long}-s if they fit into that, otherwise they will be + * {@link BigDecimal}-s. Another difference to the result of FTL literals is that instead of + * {@code HashLiteral.SequenceHash} it uses {@link SimpleHash} with {@link LinkedHashMap} as backing store, for + * efficiency. + * + * <p>This parser allows certain things that are errors in pure JSON: + * <ul> + * <li>JavaScript comments are supported</li> + * <li>Non-breaking space (nbsp) and BOM are treated as whitespace</li> + * </ul> + */ +class JSONParser { + + private static final String UNCLOSED_OBJECT_MESSAGE + = "This {...} was still unclosed when the end of the file was reached. (Look for a missing \"}\")"; + + private static final String UNCLOSED_ARRAY_MESSAGE + = "This [...] was still unclosed when the end of the file was reached. (Look for a missing \"]\")"; + + private static final BigDecimal MIN_INT_AS_BIGDECIMAL = BigDecimal.valueOf(Integer.MIN_VALUE); + private static final BigDecimal MAX_INT_AS_BIGDECIMAL = BigDecimal.valueOf(Integer.MAX_VALUE); + private static final BigDecimal MIN_LONG_AS_BIGDECIMAL = BigDecimal.valueOf(Long.MIN_VALUE); + private static final BigDecimal MAX_LONG_AS_BIGDECIMAL = BigDecimal.valueOf(Long.MAX_VALUE); + + private final String src; + private final int ln; + + private int p; + + public static TemplateModel parse(String src) throws JSONParseException { + return new JSONParser(src).parse(); + } + + /** + * @param sourceLocation Only used in error messages, maybe {@code null}. + */ + private JSONParser(String src) { + this.src = src; + this.ln = src.length(); + } + + private TemplateModel parse() throws JSONParseException { + skipWS(); + TemplateModel result = consumeValue("Empty JSON (contains no value)", p); + + skipWS(); + if (p != ln) { + throw newParseException("End-of-file was expected but found further non-whitespace characters."); + } + + return result; + } + + private TemplateModel consumeValue(String eofErrorMessage, int eofBlamePosition) throws JSONParseException { + if (p == ln) { + throw newParseException( + eofErrorMessage == null + ? "A value was expected here, but end-of-file was reached." : eofErrorMessage, + eofBlamePosition == -1 ? p : eofBlamePosition); + } + + TemplateModel result; + + result = tryConsumeString(); + if (result != null) return result; + + result = tryConsumeNumber(); + if (result != null) return result; + + result = tryConsumeObject(); + if (result != null) return result; + + result = tryConsumeArray(); + if (result != null) return result; + + result = tryConsumeTrueFalseNull(); + if (result != null) return result != TemplateNullModel.INSTANCE ? result : null; + + // Better error message for a frequent mistake: + if (p < ln && src.charAt(p) == '\'') { + throw newParseException("Unexpected apostrophe-quote character. " + + "JSON strings must be quoted with quotation mark."); + } + + throw newParseException( + "Expected either the beginning of a (negative) number or the beginning of one of these: " + + "{...}, [...], \"...\", true, false, null. Found character " + StringUtil.jQuote(src.charAt(p)) + + " instead."); + } + + private TemplateModel tryConsumeTrueFalseNull() throws JSONParseException { + int startP = p; + if (p < ln && isIdentifierStart(src.charAt(p))) { + p++; + while (p < ln && isIdentifierPart(src.charAt(p))) { + p++; + } + } + + if (startP == p) return null; + + String keyword = src.substring(startP, p); + if (keyword.equals("true")) { + return TemplateBooleanModel.TRUE; + } else if (keyword.equals("false")) { + return TemplateBooleanModel.FALSE; + } else if (keyword.equals("null")) { + return TemplateNullModel.INSTANCE; + } + + throw newParseException( + "Invalid JSON keyword: " + StringUtil.jQuote(keyword) + + ". Should be one of: true, false, null. " + + "If it meant to be a string then it must be quoted.", startP); + } + + private TemplateNumberModel tryConsumeNumber() throws JSONParseException { + if (p >= ln) { + return null; + } + char c = src.charAt(p); + boolean negative = c == '-'; + if (!(negative || isDigit(c) || c == '.')) { + return null; + } + + int startP = p; + + if (negative) { + if (p + 1 >= ln) { + throw newParseException("Expected a digit after \"-\", but reached end-of-file."); + } + char lookAheadC = src.charAt(p + 1); + if (!(isDigit(lookAheadC) || lookAheadC == '.')) { + return null; + } + p++; // Consume "-" only, not the digit + } + + long longSum = 0; + boolean firstDigit = true; + consumeLongFittingHead: do { + c = src.charAt(p); + + if (!isDigit(c)) { + if (c == '.' && firstDigit) { + throw newParseException("JSON doesn't allow numbers starting with \".\"."); + } + break consumeLongFittingHead; + } + + int digit = c - '0'; + if (longSum == 0) { + if (!firstDigit) { + throw newParseException("JSON doesn't allow superfluous leading 0-s.", p - 1); + } + + longSum = !negative ? digit : -digit; + p++; + } else { + long prevLongSum = longSum; + longSum = longSum * 10 + (!negative ? digit : -digit); + if (!negative && prevLongSum > longSum || negative && prevLongSum < longSum) { + // We had an overflow => Can't consume this digit as long-fitting + break consumeLongFittingHead; + } + p++; + } + firstDigit = false; + } while (p < ln); + + if (p < ln && isBigDecimalFittingTailCharacter(c)) { + char lastC = c; + p++; + + consumeBigDecimalFittingTail: while (p < ln) { + c = src.charAt(p); + if (isBigDecimalFittingTailCharacter(c)) { + p++; + } else if ((c == '+' || c == '-') && isE(lastC)) { + p++; + } else { + break consumeBigDecimalFittingTail; + } + lastC = c; + } + + String numStr = src.substring(startP, p); + BigDecimal bd; + try { + bd = new BigDecimal(numStr); + } catch (NumberFormatException e) { + throw new JSONParseException("Malformed number: " + numStr, src, startP, e); + } + + if (bd.compareTo(MIN_INT_AS_BIGDECIMAL) >= 0 && bd.compareTo(MAX_INT_AS_BIGDECIMAL) <= 0) { + if (NumberUtil.isIntegerBigDecimal(bd)) { + return new SimpleNumber(bd.intValue()); + } + } else if (bd.compareTo(MIN_LONG_AS_BIGDECIMAL) >= 0 && bd.compareTo(MAX_LONG_AS_BIGDECIMAL) <= 0) { + if (NumberUtil.isIntegerBigDecimal(bd)) { + return new SimpleNumber(bd.longValue()); + } + } + return new SimpleNumber(bd); + } else { + return new SimpleNumber( + longSum <= Integer.MAX_VALUE && longSum >= Integer.MIN_VALUE + ? (Number) (int) longSum + : longSum); + } + } + + private TemplateScalarModel tryConsumeString() throws JSONParseException { + int startP = p; + if (!tryConsumeChar('"')) return null; + + StringBuilder sb = new StringBuilder(); + char c = 0; + while (p < ln) { + c = src.charAt(p); + + if (c == '"') { + p++; + return new SimpleScalar(sb.toString()); // Call normally returns here! + } else if (c == '\\') { + p++; + sb.append(consumeAfterBackslash()); + } else if (c <= 0x1F) { + throw newParseException("JSON doesn't allow unescaped control characters in string literals, " + + "but found character with code (decimal): " + (int) c); + } else { + p++; + sb.append(c); + } + } + + throw newParseException("String literal was still unclosed when the end of the file was reached. " + + "(Look for missing or accidentally escaped closing quotation mark.)", startP); + } + + private TemplateSequenceModel tryConsumeArray() throws JSONParseException { + int startP = p; + if (!tryConsumeChar('[')) return null; + + skipWS(); + if (tryConsumeChar(']')) return Constants.EMPTY_SEQUENCE; + + boolean afterComma = false; + SimpleSequence elements = new SimpleSequence(_TemplateAPI.SAFE_OBJECT_WRAPPER); + do { + skipWS(); + elements.add(consumeValue(afterComma ? null : UNCLOSED_ARRAY_MESSAGE, afterComma ? -1 : startP)); + + skipWS(); + afterComma = true; + } while (consumeChar(',', ']', UNCLOSED_ARRAY_MESSAGE, startP) == ','); + return elements; + } + + private TemplateHashModelEx2 tryConsumeObject() throws JSONParseException { + int startP = p; + if (!tryConsumeChar('{')) return null; + + skipWS(); + if (tryConsumeChar('}')) return Constants.EMPTY_HASH_EX2; + + boolean afterComma = false; + Map<String, Object> map = new LinkedHashMap<>(); // Must keeps original order! + do { + skipWS(); + int keyStartP = p; + Object key = consumeValue(afterComma ? null : UNCLOSED_OBJECT_MESSAGE, afterComma ? -1 : startP); + if (!(key instanceof TemplateScalarModel)) { + throw newParseException("Wrong key type. JSON only allows string keys inside {...}.", keyStartP); + } + String strKey = null; + try { + strKey = ((TemplateScalarModel) key).getAsString(); + } catch (TemplateModelException e) { + throw new BugException(e); + } + + skipWS(); + consumeChar(':'); + + skipWS(); + map.put(strKey, consumeValue(null, -1)); + + skipWS(); + afterComma = true; + } while (consumeChar(',', '}', UNCLOSED_OBJECT_MESSAGE, startP) == ','); + return new SimpleHash(map, _TemplateAPI.SAFE_OBJECT_WRAPPER, 0); + } + + private boolean isE(char c) { + return c == 'e' || c == 'E'; + } + + private boolean isBigDecimalFittingTailCharacter(char c) { + return c == '.' || isE(c) || isDigit(c); + } + + private char consumeAfterBackslash() throws JSONParseException { + if (p == ln) { + throw newParseException("Reached the end of the file, but the escape is unclosed."); + } + + final char c = src.charAt(p); + switch (c) { + case '"': + case '\\': + case '/': + p++; + return c; + case 'b': + p++; + return '\b'; + case 'f': + p++; + return '\f'; + case 'n': + p++; + return '\n'; + case 'r': + p++; + return '\r'; + case 't': + p++; + return '\t'; + case 'u': + p++; + return consumeAfterBackslashU(); + } + throw newParseException("Unsupported escape: \\" + c); + } + + private char consumeAfterBackslashU() throws JSONParseException { + if (p + 3 >= ln) { + throw newParseException("\\u must be followed by exactly 4 hexadecimal digits"); + } + final String hex = src.substring(p, p + 4); + try { + char r = (char) Integer.parseInt(hex, 16); + p += 4; + return r; + } catch (NumberFormatException e) { + throw newParseException("\\u must be followed by exactly 4 hexadecimal digits, but was followed by " + + StringUtil.jQuote(hex) + "."); + } + } + + private boolean tryConsumeChar(char c) { + if (p < ln && src.charAt(p) == c) { + p++; + return true; + } else { + return false; + } + } + + private void consumeChar(char expected) throws JSONParseException { + consumeChar(expected, (char) 0, null, -1); + } + + private char consumeChar(char expected1, char expected2, String eofErrorHint, int eofErrorP) throws JSONParseException { + if (p >= ln) { + throw newParseException(eofErrorHint == null + ? "Expected " + StringUtil.jQuote(expected1) + + ( expected2 != 0 ? " or " + StringUtil.jQuote(expected2) : "") + + " character, but reached end-of-file. " + : eofErrorHint, + eofErrorP == -1 ? p : eofErrorP); + } + char c = src.charAt(p); + if (c == expected1 || (expected2 != 0 && c == expected2)) { + p++; + return c; + } + throw newParseException("Expected " + StringUtil.jQuote(expected1) + + ( expected2 != 0 ? " or " + StringUtil.jQuote(expected2) : "") + + " character, but found " + StringUtil.jQuote(c) + " instead."); + } + + private void skipWS() throws JSONParseException { + do { + while (p < ln && isWS(src.charAt(p))) { + p++; + } + } while (skipComment()); + } + + private boolean skipComment() throws JSONParseException { + if (p + 1 < ln) { + if (src.charAt(p) == '/') { + char c2 = src.charAt(p + 1); + if (c2 == '/') { + int eolP = p + 2; + while (eolP < ln && !isLineBreak(src.charAt(eolP))) { + eolP++; + } + p = eolP; + return true; + } else if (c2 == '*') { + int closerP = p + 3; + while (closerP < ln && !(src.charAt(closerP - 1) == '*' && src.charAt(closerP) == '/')) { + closerP++; + } + if (closerP >= ln) { + throw newParseException("Unclosed comment"); + } + p = closerP + 1; + return true; + } + } + } + return false; + } + + /** + * Whitespace as specified by JSON, plus non-breaking space (nbsp), and BOM. + */ + private static boolean isWS(char c) { + return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == 0xA0 || c == '\uFEFF'; + } + + private static boolean isLineBreak(char c) { + return c == '\r' || c == '\n'; + } + + private static boolean isIdentifierStart(char c) { + return Character.isLetter(c) || c == '_' || c == '$'; + } + + private static boolean isDigit(char c) { + return c >= '0' && c <= '9'; + } + + private static boolean isIdentifierPart(char c) { + return isIdentifierStart(c) || isDigit(c); + } + + private JSONParseException newParseException(String message) { + return newParseException(message, p); + } + + private JSONParseException newParseException(String message, int p) { + return new JSONParseException(message, src, p); + } + + static class JSONParseException extends Exception { + public JSONParseException(String message, String src, int position) { + super(createSourceCodeErrorMessage(message, src, position)); + } + + public JSONParseException(String message, String src, int position, + Throwable cause) { + super(createSourceCodeErrorMessage(message, src, position), cause); + } + + } + + private static int MAX_QUOTATION_LENGTH = 50; + + private static String createSourceCodeErrorMessage(String message, String srcCode, int position) { + int ln = srcCode.length(); + if (position < 0) { + position = 0; + } + if (position >= ln) { + return message + "\n" + + "Error location: At the end of text."; + } + + int i; + char c; + int rowBegin = 0; + int rowEnd; + int row = 1; + char lastChar = 0; + for (i = 0; i <= position; i++) { + c = srcCode.charAt(i); + if (lastChar == 0xA) { + rowBegin = i; + row++; + } else if (lastChar == 0xD && c != 0xA) { + rowBegin = i; + row++; + } + lastChar = c; + } + for (i = position; i < ln; i++) { + c = srcCode.charAt(i); + if (c == 0xA || c == 0xD) { + if (c == 0xA && i > 0 && srcCode.charAt(i - 1) == 0xD) { + i--; + } + break; + } + } + rowEnd = i - 1; + if (position > rowEnd + 1) { + position = rowEnd + 1; + } + int col = position - rowBegin + 1; + if (rowBegin > rowEnd) { + return message + "\n" + + "Error location: line " + + row + ", column " + col + ":\n" + + "(Can't show the line because it is empty.)"; + } + String s1 = srcCode.substring(rowBegin, position); + String s2 = srcCode.substring(position, rowEnd + 1); + s1 = expandTabs(s1, 8); + int ln1 = s1.length(); + s2 = expandTabs(s2, 8, ln1); + int ln2 = s2.length(); + if (ln1 + ln2 > MAX_QUOTATION_LENGTH) { + int newLn2 = ln2 - ((ln1 + ln2) - MAX_QUOTATION_LENGTH); + if (newLn2 < 6) { + newLn2 = 6; + } + if (newLn2 < ln2) { + s2 = s2.substring(0, newLn2 - 3) + "..."; + ln2 = newLn2; + } + if (ln1 + ln2 > MAX_QUOTATION_LENGTH) { + s1 = "..." + s1.substring((ln1 + ln2) - MAX_QUOTATION_LENGTH + 3); + } + } + StringBuilder res = new StringBuilder(message.length() + 80); + res.append(message); + res.append("\nError location: line ").append(row).append(", column ").append(col).append(":\n"); + res.append(s1).append(s2).append("\n"); + int x = s1.length(); + while (x != 0) { + res.append(' '); + x--; + } + res.append('^'); + + return res.toString(); + } + + private static String expandTabs(String s, int tabWidth) { + return expandTabs(s, tabWidth, 0); + } + + /** + * Replaces all tab-s with spaces in a single line. + */ + private static String expandTabs(String s, int tabWidth, int startCol) { + int e = s.indexOf('\t'); + if (e == -1) { + return s; + } + int b = 0; + StringBuilder buf = new StringBuilder(s.length() + Math.max(16, tabWidth * 2)); + do { + buf.append(s, b, e); + int col = buf.length() + startCol; + for (int i = tabWidth * (1 + col / tabWidth) - col; i > 0; i--) { + buf.append(' '); + } + b = e + 1; + e = s.indexOf('\t', b); + } while (e != -1); + buf.append(s, b, s.length()); + return buf.toString(); + } + +} diff --git a/src/manual/en_US/book.xml b/src/manual/en_US/book.xml index ffdc017..4d5fba3 100644 --- a/src/manual/en_US/book.xml +++ b/src/manual/en_US/book.xml @@ -12795,6 +12795,11 @@ grant codeBase "file:/path/to/freemarker.jar" </listitem> <listitem> + <para><link + linkend="ref_builtin_eval_json">eval_json</link></para> + </listitem> + + <listitem> <para><link linkend="ref_builtin_filter">filter</link></para> </listitem> @@ -19168,6 +19173,16 @@ Filer for positives: linkend="ref_builtin_interpret"><literal>interpret</literal> built-in</link> instead.)</para> + <important> + <para>Do not use this to evaluate JSON! For that use the <link + linkend="ref_builtin_eval_json"><literal>eval_json</literal> + built-in</link> instead. While FTL expression language looks + similar to JSON, not all JSON is valid FTL expression. Also, FTL + expressions can access variables, and call Java methods on them, + so if you <literal>?eval</literal> strings coming from untrusted + source, it can become an attack vector.</para> + </important> + <para>The evaluated expression sees the same variables (such as locals) that are visible at the place of the invocation of <literal>eval</literal>. That is, it behaves similarly as if in @@ -19184,6 +19199,75 @@ Filer for positives: built-in</link>.</para> </section> + <section xml:id="ref_builtin_eval_json"> + <title>eval_json</title> + + <indexterm> + <primary>eval_json</primary> + </indexterm> + + <indexterm> + <primary>evaluate string</primary> + </indexterm> + + <indexterm> + <primary>JSON</primary> + </indexterm> + + <note> + <para>This built-in is available since FreeMarker 2.3.31.</para> + </note> + + <para>This built-in evaluates a string as a JSON + <emphasis>expression</emphasis>, so that you can extract data from + inside it. For example, if you receive data in the + <literal>dataJson</literal> variable, but it's unfortunately just a + flat string that contains <literal>{"name": "foo", "ids": [11, + 22]}</literal>, then you can extract data from it like this:</para> + + <programlisting role="template"><#assign data = dataJson<emphasis>?eval_json</emphasis>> +<p>Name: ${data.name} +<p>Ids: +<ul> + <#list data.ids as id> + <li>${id} + </#list> +</ul></programlisting> + + <para>Ideally, you shouldn't need <literal>eval_json</literal>, + since the template should receive data already parsed (to + <literal>List</literal>-s, <literal>Map</literal>-s, Java beans, + etc.). This built-in is there as a workaround, if you can't improve + the data-model.</para> + + <para>The evaluated JSON expression doesn't have to be a JSON object + (key-value pairs), it can be any kind of JSON value, like JSON + array, JSON number, etc.</para> + + <para>The syntax understood by this built-in is a superset of + JSON:</para> + + <itemizedlist> + <listitem> + <para>Java-style comments are supported + (<literal>/*<replaceable>...</replaceable>*/</literal> and + <literal>//<replaceable>...</replaceable></literal>)</para> + </listitem> + + <listitem> + <para>BOM (byte order mark) and non-breaking space + (<quote>nbsp</quote>) are treated as whitespace (in a stricter + JSON parser they are errors of occurring around tokens).</para> + </listitem> + </itemizedlist> + + <para>No other non-JSON extras are implemented, notably, it's + impossible to refer to variables (unlike in the <link + linkend="ref_builtin_eval"><literal>eval</literal> built-in</link>). + This is important for safety, when receiving JSON from untrusted + sources.</para> + </section> + <section xml:id="ref_builtin_has_content"> <title>has_content</title> @@ -29317,6 +29401,20 @@ TemplateModel x = env.getVariable("x"); // get variable x</programlisting> <para>Release date: FIXME</para> <section> + <title>Changes on the FTL side</title> + + <itemizedlist> + <listitem> + <para>Added <literal>?eval_json</literal> to evaluate JSON given + as flat string. This was added as <literal>?eval</literal> is + routinely misused for the same purpose, which not only doesn't + work for all JSON-s, but can be a security problem. <link + linkend="ref_builtin_eval_json">See more here...</link></para> + </listitem> + </itemizedlist> + </section> + + <section> <title>Changes on the Java side</title> <itemizedlist> diff --git a/src/test/java/freemarker/core/EvalJsonBuiltInTest.java b/src/test/java/freemarker/core/EvalJsonBuiltInTest.java new file mode 100644 index 0000000..ff62a61 --- /dev/null +++ b/src/test/java/freemarker/core/EvalJsonBuiltInTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package freemarker.core; + +import java.io.IOException; + +import org.junit.Test; + +import freemarker.template.TemplateException; +import freemarker.test.TemplateTest; + +public class EvalJsonBuiltInTest extends TemplateTest { + + @Test + public void test() throws Exception { + assertOutput("${'1'?eval_json}", "1"); + assertOutput("${'1'?evalJson}", "1"); + + assertOutput("${'null'?evalJson!'-'}", "-"); + + assertOutput("<#list '{\"a\": 1e2, \"b\": null}'?evalJson as k, v>${k}=${v!'NULL'}<#sep>, </#list>", "a=100, b=NULL"); + } + +} diff --git a/src/test/java/freemarker/core/JSONParserTest.java b/src/test/java/freemarker/core/JSONParserTest.java new file mode 100644 index 0000000..dfdf5b6 --- /dev/null +++ b/src/test/java/freemarker/core/JSONParserTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package freemarker.core; + +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; + +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; + +import org.junit.Assert; +import org.junit.Test; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import freemarker.template.TemplateModel; +import freemarker.template.TemplateModelException; +import freemarker.template.utility.DeepUnwrap; + +public class JSONParserTest { + + @Test + public void testObjects() throws JSONParser.JSONParseException { + assertEquals(ImmutableMap.of("a", 1, "b", 2), JSONParser.parse("{\"a\": 1, \"b\": 2}")); + assertEquals(Collections.emptyMap(), JSONParser.parse("{}")); + try { + JSONParser.parse("{1: 1}"); + fail(); + } catch (JSONParser.JSONParseException e) { + assertThat(e.getMessage(), containsString("string key")); + } + } + + @Test + public void testLists() throws JSONParser.JSONParseException { + assertEquals(ImmutableList.of(1, 2), JSONParser.parse("[1, 2]")); + assertEquals(Collections.emptyList(), JSONParser.parse("[]")); + } + + @Test + public void testStrings() throws JSONParser.JSONParseException { + assertEquals("", JSONParser.parse("\"\"")); + assertEquals(" ", JSONParser.parse("\" \"")); + assertEquals("'", JSONParser.parse("\"'\"")); + assertEquals("foo", JSONParser.parse("\"foo\"")); + assertEquals("\" \\ / \b \f \n \r \t \ufeff", + JSONParser.parse( + "\"" + + "\\\" \\\\ \\/ \\b \\f \\n \\r \\t \\uFEFF" + + "\"")); + } + + @Test + public void testNumbers() throws JSONParser.JSONParseException { + assertEquals(0, JSONParser.parse("0")); + assertEquals(123, JSONParser.parse("123")); + assertEquals(-123, JSONParser.parse("-123")); + assertNotEquals(123L, JSONParser.parse("123")); + assertEquals(2147483647, JSONParser.parse("2147483647")); + assertEquals(2147483648L, JSONParser.parse("2147483648")); + assertEquals(-2147483648, JSONParser.parse("-2147483648")); + assertEquals(-2147483649L, JSONParser.parse("-2147483649")); + assertEquals(-123, JSONParser.parse("-1.23E2")); + assertEquals(new BigDecimal("1.23"), JSONParser.parse("1.23")); + assertEquals(new BigDecimal("-1.23"), JSONParser.parse("-1.23")); + assertEquals(new BigDecimal("12.3"), JSONParser.parse("1.23E1")); + assertEquals(new BigDecimal("0.123"), JSONParser.parse("123E-3")); + } + + @Test + public void testKeywords() throws JSONParser.JSONParseException { + assertNull(JSONParser.parse("null")); + assertEquals(true, JSONParser.parse("true")); + assertEquals(false, JSONParser.parse("false")); + try { + JSONParser.parse("NULL"); + fail(); + } catch (JSONParser.JSONParseException e) { + assertThat(e.getMessage(), containsString("quoted")); + } + } + + @Test + public void testBlockComments() throws JSONParser.JSONParseException { + assertEquals(ImmutableList.of(1, 2), JSONParser.parse("/**/[/**/1/**/, /**/2/**/]/**/")); + assertEquals(ImmutableList.of(1, 2), JSONParser.parse("/*x*/[/*x*/1/*x*/, /*x*/2/*x*/]/*x*/")); + assertEquals(ImmutableList.of(1), JSONParser.parse(" /*x*/ /**//**/ [ /*x*/ /*\n*//***/ 1 ]")); + try { + JSONParser.parse("/*"); + fail(); + } catch (JSONParser.JSONParseException e) { + assertThat(e.getMessage(), containsString("Unclosed comment")); + } + try { + JSONParser.parse("[/*]"); + fail(); + } catch (JSONParser.JSONParseException e) { + assertThat(e.getMessage(), containsString("Unclosed comment")); + } + } + + @Test + public void testLineComments() throws JSONParser.JSONParseException { + assertEquals(ImmutableList.of(1, 2), JSONParser.parse("//c1\n[ //c2\n1, //c3\n 2//c5\n] //c4")); + assertEquals(ImmutableList.of(1, 2), JSONParser.parse("// c1\n//\r// c2\r\n// c3\r\n[ 1, 2 ]//")); + assertEquals(ImmutableList.of(1, 2), JSONParser.parse("[1, 2]\n//\n")); + } + + @Test + public void testWhitespace() throws JSONParser.JSONParseException { + assertEquals(ImmutableList.of(1, 2), JSONParser.parse(" [ 1 ,\n2 ] ")); + assertEquals(ImmutableList.of(1, 2), JSONParser.parse("\uFEFF[\u00A01\u00A0,2]")); + } + + @Test + public void testMixed() throws JSONParser.JSONParseException { + LinkedHashMap<String, Object> m = new LinkedHashMap<>(); + m.put("x", 1); + m.put("y", null); + assertEquals( + ImmutableList.of( + ImmutableMap.of("a", Collections.emptyMap()), + ImmutableMap.of("b", + Arrays.asList( + m, + true, + null + )) + ), + JSONParser.parse("" + + "[\n" + + "{\"a\":{}},\n" + + "{\"b\":\n" + + "[" + + "{\"x\":1, \"y\": null}," + + "true," + + "null" + + "] // comment\n" + + "}\n" + + "]")); + } + + private static void assertEquals(Object expected, TemplateModel actual) { + try { + Assert.assertEquals(expected, DeepUnwrap.unwrap(actual)); + } catch (TemplateModelException e) { + throw new BugException(e); + } + } + +} \ No newline at end of file
