manner commented on code in PR #29520: URL: https://github.com/apache/flink/pull/29520#discussion_r4514121575
########## flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantToXmlWriter.java: ########## @@ -0,0 +1,784 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.runtime.functions.XmlVariantNames.SchemaPrefix; +import org.apache.flink.types.variant.Variant; + +import org.w3c.dom.DOMException; +import org.w3c.dom.Document; + +import javax.annotation.Nullable; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamWriter; + +import java.io.StringWriter; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.apache.flink.table.runtime.functions.XmlVariantNames.ATTRIBUTE_PREFIX; +import static org.apache.flink.table.runtime.functions.XmlVariantNames.ORDER_KEY; +import static org.apache.flink.table.runtime.functions.XmlVariantNames.TEXT_KEY; +import static org.apache.flink.table.runtime.functions.XmlVariantNames.XSI_NIL; +import static org.apache.flink.table.runtime.functions.XmlVariantNames.XSI_TYPE; +import static org.apache.flink.types.variant.BinaryVariantUtil.TIMESTAMP_FORMATTER; +import static org.apache.flink.types.variant.BinaryVariantUtil.TIMESTAMP_LTZ_FORMATTER; +import static org.apache.flink.types.variant.BinaryVariantUtil.TIME_FORMATTER; +import static org.apache.flink.util.Preconditions.checkArgument; +import static org.apache.flink.util.Preconditions.checkState; + +/** + * Writes a {@link Variant} as an XML string for {@code XML_STRING}. It reverses the mapping of + * {@link XmlToVariantParser}, so that parsing the XML string of a variant from {@code PARSE_XML} + * gives the same variant. The mapping is described in the documentation of {@code XML_STRING}. + * + * <p>The content of an element is written in the order of its {@code #} field. Content without a + * position in the {@code #} field follows in the order of its keys. A variant {@code NULL}, or a + * {@code NULL} text, is written as {@code xsi:nil="true"}, and a scalar other than a string or a + * UUID gets an {@code xsi:type}, e.g. {@code xsi:type="xs:int"}. If the document uses the xsi or xs + * prefix, the root element declares it with the namespace of XML Schema, unless the variant + * declares it on the root element itself. + * + * <p>The XML writer of the JDK writes the tags and escapes the text. It writes tabs and line breaks + * in attribute values, and carriage returns in text, as they are, so a parser reads them back as + * spaces and line feeds. + * + * <p>A variant fails with an {@link IllegalArgumentException} if it has no XML representation: + * + * <ul> + * <li>it isn't an object with a single field for the root element, or the root is an array, + * <li>a key isn't an XML name that {@code PARSE_XML} accepts, + * <li>a text or an attribute value has a character that XML 1.0 doesn't allow, e.g. U+0000, + * <li>an array is nested in an array, + * <li>an {@code @} field holds an object, an array, or a {@code NULL}, or a {@code $} field holds + * an object, + * <li>a {@code NULL} text isn't the only content of its element, or an element has an {@code + * @xsi:nil} field that is true and content, + * <li>a {@code #} field isn't an object, doesn't have one integer position for each item of a + * key, or has a position more than once, + * <li>an element would need two {@code xsi:type} or two {@code xsi:nil} attributes, e.g. an + * {@code @xsi:type} field and a typed text, or an {@code @xsi:nil} field and a {@code NULL} + * text. + * </ul> + */ +@Internal +public final class VariantToXmlWriter { + + private static final Set<Variant.Type> INTEGER_TYPES = + EnumSet.of( + Variant.Type.TINYINT, + Variant.Type.SMALLINT, + Variant.Type.INT, + Variant.Type.BIGINT); + + // The XML writer of the JDK, regardless of other StAX implementations on the classpath. Its + // factory creates a new writer for each call and keeps no state of it, so it can be shared. + private static final XMLOutputFactory OUTPUT_FACTORY = XMLOutputFactory.newDefaultFactory(); + + // Any character outside of the Char production of XML 1.0, see + // https://www.w3.org/TR/xml/#charsets. + private static final Pattern NON_XML_CHAR = + Pattern.compile("[^\t\n\r\u0020-\uD7FF\uE000-\uFFFD\\x{10000}-\\x{10FFFF}]"); + + // The DOM document that names are checked with. It isn't thread-safe, so each thread has its + // own. + private static final ThreadLocal<Document> NAME_DOCUMENT = + ThreadLocal.withInitial(VariantToXmlWriter::newDocument); + + // The document that is written, and the schema prefixes that it uses. + private final StringWriter output = new StringWriter(); + private final XMLStreamWriter xml; + private final Set<SchemaPrefix> usedSchemaPrefixes = EnumSet.noneOf(SchemaPrefix.class); + + private VariantToXmlWriter() { + try { + xml = OUTPUT_FACTORY.createXMLStreamWriter(output); + } catch (XMLStreamException e) { + // Only the message, since the location of the exception isn't serializable. + throw new IllegalStateException(e.getMessage()); + } + } + + public static String write(Variant variant) { + final String rootName = rootName(variant); + final Variant root = variant.getField(rootName); + checkRootIsNotArray(rootName, root); + return new VariantToXmlWriter().writeDocument(rootName, root); + } + + /** Returns the name of the root element, the single field of the variant. */ + private static String rootName(Variant variant) { + checkArgument( + variant.isObject(), + "The variant must be an object with a single field for the root element, but it " + + "is of type %s.", + variant.getType()); + final List<String> fieldNames = variant.getFieldNames(); + checkArgument( + fieldNames.size() == 1, + "The variant must be an object with a single field for the root element, but it " + + "has %s fields.", + fieldNames.size()); + return fieldNames.get(0); + } + + private static void checkRootIsNotArray(String rootName, Variant root) { + checkArgument( + !root.isArray(), + "The root element '%s' can't be an array, since a document has only one root " + + "element.", + rootName); + } + + // -------------------------------------------------------------------------------------------- + + private String writeDocument(String rootName, Variant root) { + try { + writeElement(rootName, root); + // Closes the tag of a nil root element, which an empty element leaves open. + xml.writeEndDocument(); + xml.close(); + } catch (XMLStreamException e) { + // Only the message, since the location of the exception isn't serializable. + throw new IllegalStateException(e.getMessage()); + } + insertIntoRootTag(rootName, schemaPrefixDeclarations(root)); + return output.toString(); + } + + /** + * Returns the declarations of the schema prefixes that the document uses, without the ones that + * the root element declares itself. A declaration on another element only applies within that + * element, so the root element needs its own. + */ + private String schemaPrefixDeclarations(Variant root) { + final StringBuilder declarations = new StringBuilder(); + for (SchemaPrefix schemaPrefix : usedSchemaPrefixes) { + if (!declaresOnRoot(root, schemaPrefix)) { + declarations.append( + String.format( + " %s=\"%s\"", schemaPrefix.declaration, schemaPrefix.namespace)); + } + } + return declarations.toString(); + } + + private static boolean declaresOnRoot(Variant root, SchemaPrefix schemaPrefix) { + return root.isObject() + && root.getField(ATTRIBUTE_PREFIX + schemaPrefix.declaration) != null; + } + + /** + * Inserts the attributes after the name of the root element. They are inserted at the end, + * since only then it is known which prefixes the document uses. The document starts with the + * start tag of the root element, since it has no XML declaration. + */ + private void insertIntoRootTag(String rootName, String attributes) { + final StringBuffer document = output.getBuffer(); + final String rootTagStart = "<" + rootName; + checkState( + document.indexOf(rootTagStart) == 0, + "The document doesn't start with %s.", + rootTagStart); + document.insert(rootTagStart.length(), attributes); + } + + // -------------------------------------------------------------------------------------------- + // Elements + // -------------------------------------------------------------------------------------------- + + /** Writes a single item of a key, which isn't an array. */ + private void writeElement(String name, Variant value) throws XMLStreamException { + if (value.isNull()) { + writeNilElement(name); + } else if (value.isObject()) { + writeObjectElement(name, value); + } else { + writeScalarElement(name, value); + } + } + + /** A variant {@code NULL} is nil, since an empty element is an empty string. */ + private void writeNilElement(String name) throws XMLStreamException { + writeEmptyElement(name); + writeXsiNil(); + } + + private void writeScalarElement(String name, Variant scalar) throws XMLStreamException { + writeStartElement(name); + writeXsiType(scalar); + writeText(name, scalar); + writeEndElement(); + } + + /** + * Writes an object as an element with the {@code @} fields as attributes, the {@code $} field + * as text, and the other fields as child elements. + */ + private void writeObjectElement(String name, Variant element) throws XMLStreamException { + final ElementKeys keys = new ElementKeys(element); + final List<ContentItem> content = content(name, element, keys.contentKeys); + if (hasNullText(content)) { + writeNilElementWithAttributes(name, element, keys.attributeKeys); + } else { + writeElementWithContent(name, element, keys.attributeKeys, content); + } + } + + /** Like in {@code PARSE_XML}, a nil element with attributes has a null text and no content. */ + private static boolean hasNullText(List<ContentItem> content) { + final @Nullable Variant text = onlyText(content); + return text != null && text.isNull(); + } + + private void writeNilElementWithAttributes( + String name, Variant element, List<String> attributeKeys) throws XMLStreamException { + checkNoAttributeField(name, element, XSI_NIL, "a null text"); + writeEmptyElement(name); + writeFields(name, element, attributeKeys); + writeXsiNil(); + } + + private void writeElementWithContent( + String name, Variant element, List<String> attributeKeys, List<ContentItem> content) + throws XMLStreamException { + checkIsNotNil(name, element, content); + writeStartElement(name); + // Attributes come first, since the XML writer writes them into the start tag. + writeFields(name, element, attributeKeys); + writeXsiTypeOfText(name, element, content); + for (ContentItem item : content) { + writeField(name, item.key, item.value); + } + writeEndElement(); + } + + /** Like in {@code PARSE_XML}, only the text of an element without child elements is typed. */ + private void writeXsiTypeOfText(String name, Variant element, List<ContentItem> content) + throws XMLStreamException { + final @Nullable Variant text = onlyText(content); + if (text != null && xsiType(text) != null) { + checkNoAttributeField(name, element, XSI_TYPE, "a typed text"); + writeXsiType(text); + } + } + + /** {@code PARSE_XML} drops the content of a nil element, so it can't have content. */ + private static void checkIsNotNil( + String elementName, Variant element, List<ContentItem> content) { + final @Nullable Variant xsiNil = element.getField(ATTRIBUTE_PREFIX + XSI_NIL); + checkArgument( + content.isEmpty() || xsiNil == null || !isTrue(xsiNil), + "The element '%s' has an '%s%s' field that is true and content, but an element " + + "with %s can't have content.", + elementName, + ATTRIBUTE_PREFIX, + XSI_NIL, + XSI_NIL); + } + + /** Like in {@code PARSE_XML}, xsi:nil is true for the values true and 1. */ + private static boolean isTrue(Variant xsiNil) { + if (!isNonNullScalar(xsiNil)) { + return false; + } + final String value = lexicalForm(xsiNil).trim(); + return value.equals("true") || value.equals("1"); + } + + /** An element can't have an {@code @} field for an attribute that the writer adds as well. */ + private static void checkNoAttributeField( + String elementName, Variant element, String attribute, String reason) { + checkArgument( + element.getField(ATTRIBUTE_PREFIX + attribute) == null, + "The element '%s' has an '%s%s' field and %s, but an element can have only one %s " + + "attribute.", + elementName, + ATTRIBUTE_PREFIX, + attribute, + reason, + attribute); + } + + // -------------------------------------------------------------------------------------------- + // Fields + // -------------------------------------------------------------------------------------------- + + private void writeFields(String elementName, Variant element, List<String> keys) + throws XMLStreamException { + for (String key : keys) { + writeField(elementName, key, element.getField(key)); + } + } + + /** + * Writes a field of an element by its key: an {@code @} field as an attribute, the {@code $} + * field as text, and any other field as a child element. + */ + private void writeField(String elementName, String key, Variant value) + throws XMLStreamException { + if (key.startsWith(ATTRIBUTE_PREFIX)) { + writeAttributeField(elementName, key.substring(ATTRIBUTE_PREFIX.length()), value); + } else if (key.equals(TEXT_KEY)) { + writeText(elementName, value); + } else { + writeElement(key, value); + } + } + + private void writeAttributeField(String elementName, String name, Variant value) + throws XMLStreamException { + checkAttributeName(name); + checkIsAttributeValue(elementName, name, value); + checkXmlChars(elementName, name, value); + writeAttribute(name, lexicalForm(value)); + } + + private static void checkIsAttributeValue(String elementName, String name, Variant value) { + checkArgument( + isNonNullScalar(value), + "The attribute '%s' of the element '%s' can't be of type %s.", + name, + elementName, + value.getType()); + } + + private void writeXsiNil() throws XMLStreamException { + writeAttribute(XSI_NIL, "true"); + } + + private void writeXsiType(Variant scalar) throws XMLStreamException { + final @Nullable String xsiType = xsiType(scalar); + if (xsiType != null) { + writeAttribute(XSI_TYPE, SchemaPrefix.XS.prefix + xsiType); + } + } + + // -------------------------------------------------------------------------------------------- + // Writing to the XML writer, which writes the tags and escapes the text + // -------------------------------------------------------------------------------------------- + + private void writeStartElement(String name) throws XMLStreamException { + checkElementName(name); + noteSchemaPrefix(name); + xml.writeStartElement(name); + } + + private void writeEmptyElement(String name) throws XMLStreamException { + checkElementName(name); + noteSchemaPrefix(name); + xml.writeEmptyElement(name); + } + + private void writeEndElement() throws XMLStreamException { + xml.writeEndElement(); + } + + /** The name is checked by the caller, since the writer adds xsi:nil and xsi:type itself. */ + private void writeAttribute(String name, String value) throws XMLStreamException { Review Comment: `XMLStreamWriter` had the limitation that it always escaped `&` in attribute values, so it was impossible to correctly write a character reference such as ` ` into the XML. I replaced the `XMLStreamWriter` with our own escaping that writes tab, LF and CR as character references. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
