Author: rwhitcomb
Date: Wed Nov 5 21:47:40 2014
New Revision: 1636971
URL: http://svn.apache.org/r1636971
Log:
PIVOT-960: Add macro processing capability to JSONSerializer.
This provides a very general way to use constant definitions inside a style
sheet (for instance) or to compose values from parts defined separately.
Macros are defined used "#define NAME VALUE" and undefined with "#undef NAME".
Macros are expanded using "${NAME}" and can be recursively defined (that is,
one macro is defined in terms of another, such as:
#define TWO 2
#define ZERO 0
#define TWENTY ${TWO}${ZERO}
...
padding : ${TWENTY}
will expand to:
padding : 20
Updating the JSONSerializerTest to more thoroughly test the various macro
definitions and expansions, using an update "map.json" with a number of
macros defined in it.
The main program is "MacroReader.java" which is just a Reader that can be
combined with any other Reader to add macro capabilities to any other text
stream (such as XML, CSV, etc). For now we have only added it to
JSONSerializer.
Added:
pivot/trunk/core/src/org/apache/pivot/serialization/MacroReader.java
Modified:
pivot/trunk/core/src/org/apache/pivot/json/JSONSerializer.java
pivot/trunk/core/test/org/apache/pivot/json/test/JSONSerializerTest.java
pivot/trunk/core/test/org/apache/pivot/json/test/map.json
Modified: pivot/trunk/core/src/org/apache/pivot/json/JSONSerializer.java
URL:
http://svn.apache.org/viewvc/pivot/trunk/core/src/org/apache/pivot/json/JSONSerializer.java?rev=1636971&r1=1636970&r2=1636971&view=diff
==============================================================================
--- pivot/trunk/core/src/org/apache/pivot/json/JSONSerializer.java (original)
+++ pivot/trunk/core/src/org/apache/pivot/json/JSONSerializer.java Wed Nov 5
21:47:40 2014
@@ -42,6 +42,7 @@ import org.apache.pivot.collections.Map;
import org.apache.pivot.collections.Sequence;
import org.apache.pivot.io.EchoReader;
import org.apache.pivot.io.EchoWriter;
+import org.apache.pivot.serialization.MacroReader;
import org.apache.pivot.serialization.SerializationException;
import org.apache.pivot.serialization.Serializer;
import org.apache.pivot.util.ListenerList;
@@ -248,17 +249,18 @@ public class JSONSerializer implements S
// Move to the first character
LineNumberReader lineNumberReader = new LineNumberReader(reader);
- c = lineNumberReader.read();
+ MacroReader macroReader = new MacroReader(lineNumberReader);
+ c = macroReader.read();
// Ignore BOM (if present)
if (c == 0xFEFF) {
- c = lineNumberReader.read();
+ c = macroReader.read();
}
// Read the root value
Object object;
try {
- object = readValue(lineNumberReader, type);
+ object = readValue(macroReader, type);
} catch (SerializationException exception) {
System.err.println("An error occurred while processing input at
line number "
+ (lineNumberReader.getLineNumber() + 1));
@@ -292,7 +294,7 @@ public class JSONSerializer implements S
} else if (c == '{') {
object = readMapValue(reader, typeArgument);
} else {
- throw new SerializationException("Unexpected character in input
stream.");
+ throw new SerializationException("Unexpected character in input
stream: '" + (char)c + "'");
}
return object;
@@ -333,7 +335,7 @@ public class JSONSerializer implements S
c = reader.read();
}
} else {
- throw new SerializationException("Unexpected character in
input stream.");
+ throw new SerializationException("Unexpected character in
input stream: '" + (char)c + "'");
}
}
}
@@ -347,7 +349,7 @@ public class JSONSerializer implements S
while (c != -1 && i < n) {
if (nullString.charAt(i) != c) {
- throw new SerializationException("Unexpected character in
input stream.");
+ throw new SerializationException("Unexpected character in
input stream: '" + (char)c + "'");
}
c = reader.read();
@@ -494,7 +496,7 @@ public class JSONSerializer implements S
while (c != -1 && i < n) {
if (text.charAt(i) != c) {
- throw new SerializationException("Unexpected character in
input stream.");
+ throw new SerializationException("Unexpected character in
input stream: '" + (char)c + "'");
}
c = reader.read();
@@ -612,7 +614,7 @@ public class JSONSerializer implements S
throw new SerializationException("Unexpected end of input
stream.");
} else {
if (c != ']') {
- throw new SerializationException("Unexpected character in
input stream.");
+ throw new SerializationException("Unexpected character in
input stream: '" + (char)c + "'");
}
}
}
@@ -764,7 +766,7 @@ public class JSONSerializer implements S
skipWhitespaceAndComments(reader);
if (c != ':') {
- throw new SerializationException("Unexpected character in
input stream.");
+ throw new SerializationException("Unexpected character in
input stream: '" + (char)c + "'");
}
// Move to the first character after ':'
@@ -795,7 +797,7 @@ public class JSONSerializer implements S
throw new SerializationException("Unexpected end of input
stream.");
} else {
if (c != '}') {
- throw new SerializationException("Unexpected character in
input stream.");
+ throw new SerializationException("Unexpected character in
input stream: '" + (char)c + "'");
}
}
}
Added: pivot/trunk/core/src/org/apache/pivot/serialization/MacroReader.java
URL:
http://svn.apache.org/viewvc/pivot/trunk/core/src/org/apache/pivot/serialization/MacroReader.java?rev=1636971&view=auto
==============================================================================
--- pivot/trunk/core/src/org/apache/pivot/serialization/MacroReader.java (added)
+++ pivot/trunk/core/src/org/apache/pivot/serialization/MacroReader.java Wed
Nov 5 21:47:40 2014
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pivot.serialization;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayDeque;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Queue;
+
+
+/**
+ * This is a {@link Reader} that can be instantiated inline with any other
+ * <tt>Reader</tt> to provide macro capabilities.
+ * <p> We recognize <code>#define <i>NAME value</i></code> as definitions
+ * as well as <code>#undef <i>NAME</i></code> to remove a previous definition.
+ * <p> The macro name must correspond to the Unicode naming conventions (see
+ * {@link Character#isUnicodeIdentifierStart} and {@link
Character#isUnicodeIdentifierPart}).
+ * <p> Macro substitutions are recognized as <code>${<i>NAME</i>}</code>
anywhere
+ * in the underlying stream. Nested macros are supported, and are expanded at
the
+ * point of definition, if defined, or at the point of expansion if defined
later.
+ */
+public class MacroReader extends Reader {
+ private Reader in;
+
+ /** The map of our defined variables and their values. */
+ private Map<String, String> variableMap = new HashMap<>();
+
+ /** The lookahead queue, set either by one-character lookahead (such as
+ * while recognizing "$NAME") or from macro expansion.
+ */
+ private Queue<Integer> lookaheadQueue = new ArrayDeque<>();
+
+ /** The previous character read. */
+ private int lastCh = -1;
+
+ public MacroReader(Reader reader) {
+ this.in = reader;
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+
+ private void queue(int ch) {
+ if (ch != -1) {
+ lookaheadQueue.add(ch);
+ }
+ }
+
+ private void queue(String str) {
+ for (int i = 0; i < str.length(); i++) {
+ lookaheadQueue.add(str.codePointAt(i));
+ }
+ }
+
+ /**
+ * Parse out the next word in the stream (according to Unicode
+ * Identifier semantics) as the macro name, skipping leading whitespace.
+ */
+ private String getNextWord() throws IOException {
+ StringBuilder buf = new StringBuilder();
+ int ch;
+ while ((ch = getNextChar(true)) != -1 && Character.isWhitespace(ch)) {
+ ;
+ }
+ if (ch != -1) {
+ buf.append((char)ch);
+ while ((ch = getNextChar(true)) != -1 &&
+ ((buf.length() == 0 &&
Character.isUnicodeIdentifierStart(ch)) ||
+ (buf.length() > 0 &&
Character.isUnicodeIdentifierPart(ch)))) {
+ buf.append((char)ch);
+ }
+ // Re-queue the character that terminated the word
+ queue(ch);
+ }
+ return buf.toString();
+ }
+
+ private void skipToEol() throws IOException {
+ int ch;
+ while ((ch = getNextChar(true)) != -1 && ch != '\n') {
+ ;
+ }
+ }
+
+ /**
+ * Get the next character in the input stream, either from the
+ * {@link #lookaheadQueue} if anything is queued, or by reading
+ * from the underlying {@link Reader}.
+ * <p> This is the heart of the processing that handles both
+ * macro definition and expansion.
+ * @param handleMacros set to <tt>false</tt> only when
+ * invoking this method recursively
+ * to ignore unknown macro commands
+ * or undefined macros
+ */
+ private int getNextChar(boolean handleMacros) throws IOException {
+ int ret = -1;
+ if (!lookaheadQueue.isEmpty()) {
+ ret = lookaheadQueue.poll().intValue();
+ }
+ else {
+ ret = in.read();
+ }
+ // Check for macro define or undefine (starting with "#"
+ // at the beginning of a line) (unless we're recursing to
+ // skip an unknown declaration keyword).
+ if (ret == '#' && lastCh == '\n' && handleMacros) {
+ String keyword = getNextWord();
+ if (keyword.equalsIgnoreCase("undef")) {
+ String name = getNextWord();
+ skipToEol();
+ variableMap.remove(name);
+ return getNextChar(true);
+ }
+ else if (!keyword.equalsIgnoreCase("define")) {
+ // Basically ignore any commands we don't understand
+ // by simply queueing the text back to be read again
+ // but with the flag set to ignore this command (so
+ // we don't get into infinite recursion!)
+ queue(ret);
+ queue(keyword);
+ queue(' ');
+ return getNextChar(false);
+ }
+ // Define a macro
+ String name = getNextWord();
+ StringBuilder buf = new StringBuilder();
+ int ch;
+ while ((ch = getNextChar(true)) != -1 &&
Character.isWhitespace(ch) && ch != '\\' && ch != '\n') {
+ ;
+ }
+ queue(ch);
+ do {
+ while ((ch = getNextChar(true)) != -1 && ch != '\\' && ch !=
'\n') {
+ buf.append((char)ch);
+ }
+ // Check for line continuation character
+ if (ch == '\\') {
+ int next = getNextChar(true);
+ if (next == '\n') {
+ buf.append((char)next);
+ }
+ else {
+ buf.append((char)ch);
+ buf.append((char)next);
+ }
+ }
+ } while (ch != -1 && ch != '\n');
+ variableMap.put(name, buf.toString());
+ return getNextChar(true);
+ }
+ else if (ret == '$' && handleMacros) {
+ // Check for macro expansion
+ // Note: this allows for nested expansion
+ int next = getNextChar(true);
+ if (next == '{') {
+ // Beginning of macro expansion
+ StringBuilder buf = new StringBuilder();
+ int ch;
+ while ((ch = getNextChar(true)) != -1 && ch != '}') {
+ buf.append((char)ch);
+ }
+ String expansion = variableMap.get(buf.toString());
+ if (expansion == null) {
+ queue(ret);
+ queue(next);
+ queue(buf.toString());
+ queue(ch);
+ ret = getNextChar(false);
+ }
+ else {
+ queue(expansion);
+ ret = getNextChar(true);
+ }
+ }
+ else {
+ queue(next);
+ }
+ }
+ return (lastCh = ret);
+ }
+
+ @Override
+ public int read(char[] cbuf, int off, int len) throws IOException {
+ int read = -1;
+ for (int i = 0; i < len; i++) {
+ int ch = getNextChar(true);
+ if (ch == -1) {
+ break;
+ }
+ read = i;
+ cbuf[off + i] = (char)ch;
+ }
+ return (read == -1) ? read : read + 1;
+ }
+
+}
Modified:
pivot/trunk/core/test/org/apache/pivot/json/test/JSONSerializerTest.java
URL:
http://svn.apache.org/viewvc/pivot/trunk/core/test/org/apache/pivot/json/test/JSONSerializerTest.java?rev=1636971&r1=1636970&r2=1636971&view=diff
==============================================================================
--- pivot/trunk/core/test/org/apache/pivot/json/test/JSONSerializerTest.java
(original)
+++ pivot/trunk/core/test/org/apache/pivot/json/test/JSONSerializerTest.java
Wed Nov 5 21:47:40 2014
@@ -133,10 +133,20 @@ public class JSONSerializerTest {
jsonSerializer.getJSONSerializerListeners().add(jsonSerializerListener);
Object o1 =
jsonSerializer.readObject(getClass().getResourceAsStream("map.json"));
+ assertEquals(JSON.get(o1, "a"), 100);
+ assertEquals(JSON.get(o1, "b"), "Hello");
+ assertEquals(JSON.get(o1, "c"), false);
assertEquals(JSON.get(o1, "e.g"), 5);
+ assertEquals(JSON.get(o1, "i.a"), 200);
+ assertEquals(JSON.get(o1, "i.c"), true);
jsonSerializer.getJSONSerializerListeners().remove(jsonSerializerListener);
Object o2 =
jsonSerializer.readObject(getClass().getResourceAsStream("map.json"));
+ assertEquals(JSON.get(o2, "k[1].a"), 10);
+ assertEquals(JSON.get(o2, "k[2].a"), 100);
+ assertEquals(JSON.get(o2, "k[2].b"), 200);
+ assertEquals(JSON.get(o2, "k[2].c"), "300");
+ assertEquals(JSON.get(o2, "j"), 200);
assertTrue(o1.equals(o2));
Modified: pivot/trunk/core/test/org/apache/pivot/json/test/map.json
URL:
http://svn.apache.org/viewvc/pivot/trunk/core/test/org/apache/pivot/json/test/map.json?rev=1636971&r1=1636970&r2=1636971&view=diff
==============================================================================
--- pivot/trunk/core/test/org/apache/pivot/json/test/map.json (original)
+++ pivot/trunk/core/test/org/apache/pivot/json/test/map.json Wed Nov 5
21:47:40 2014
@@ -14,16 +14,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-{ a: 100,
- b: "Hello",
- c: false,
- d: ["1", "2", "3"],
+// Test some of the new macro capabilities
+# define TWO 2
+# define THREE 3
+#define TWENTY ${TWO}${ZERO}
+#define THIRTY ${THREE}${ZERO}
+#define TWO_HUNDRED ${TWO}${ZERO}${ZERO}
+#define THREE_HUNDRED ${THREE}${ZERO}${ZERO}
+# define ZERO 0
+# define A_B_C a: 1${ZERO}${ZERO},\
+ b: "Hello",\
+ c: false
+
+{ ${A_B_C},
+ d: ["1", "2", "${THREE}"],
e: {f: 4, g: 5, h: 6},
- "i": { a: 200, b: "Goodbye", c: true},
- j: 200,
+ "i": { a: ${TWO_HUNDRED}, b: "Goodbye", c: true},
+ j: ${TWO_HUNDRED},
k: [
- {a:1, b:2, c:"3"},
- {a:10, b:20, c:"30"},
- {a:100, b:200, c:"300"}
+ {a:1, b:2, c:"${THREE}"},
+ {a:1${ZERO}, b:${TWENTY}, c:"${THIRTY}"},
+ {a:1${ZERO}${ZERO}, b:${TWO_HUNDRED}, c:"${THREE_HUNDRED}"}
]
}
