Tim Starling has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/335596 )
Change subject: Add XML infoset coercion to DOMBuilder ...................................................................... Add XML infoset coercion to DOMBuilder * In DOMBuilder, encode names that are invalid in XML * For performance, trigger encoding names only when an exception is caught. * Invert the encoding in TestFormatter, and optionally in HtmlFormatter * Re-enable tests that now pass Change-Id: Iddfa497e07636693baa280575b086f46adefe002 --- M src/DOM/DOMBuilder.php M src/GenerateDataFiles.php M src/HTMLData.php M src/Serializer/HtmlFormatter.php M src/Serializer/TestFormatter.php M tests/phpunit/TreeBuilderTest.php 6 files changed, 238 insertions(+), 34 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/libs/RemexHtml refs/changes/96/335596/1 diff --git a/src/DOM/DOMBuilder.php b/src/DOM/DOMBuilder.php index 5519e17..e4de294 100644 --- a/src/DOM/DOMBuilder.php +++ b/src/DOM/DOMBuilder.php @@ -10,14 +10,15 @@ * A TreeHandler which constructs a DOMDocument */ class DOMBuilder implements TreeHandler { - private $doc; - private $errorCallback; - private $isFragment; - public $doctypeName; public $public; public $system; public $quirks; + + private $doc; + private $errorCallback; + private $isFragment; + private $coerced; /** * @param callable|null $errorCallback A function which is called on parse errors @@ -45,6 +46,16 @@ } } + /** + * Returns true if the document was coerced due to libxml limitations. We + * follow HTML 5.1 ยง 8.2.7 "Coercing an HTML DOM into an infoset". + * + * @return bool + */ + public function isCoerced() { + return $this->coerced; + } + public function startDocument( $fragmentNamespace, $fragmentName ) { $impl = new \DOMImplementation; $this->isFragment = $fragmentNamespace !== null; @@ -53,9 +64,10 @@ private function createDocument( $doctypeName = null, $public = null, $system = null ) { $impl = new \DOMImplementation; - if ( $doctypeName === null - || $doctypeName === '' // libxml limitation, causes test failures - ) { + if ( $doctypeName === '' ) { + $this->coerced = true; + $doc = $impl->createDocument( null, null ); + } elseif ( $doctypeName === null ) { $doc = $impl->createDocument( null, null ); } else { $doctype = $impl->createDocumentType( $doctypeName, $public, $system ); @@ -82,10 +94,31 @@ $parent->insertBefore( $node, $refNode ); } + /** + * Replace unsupported characters with a code of the form U123456. + * + * @param string $name + * @return string + */ + private function coerceName( $name ) { + $coercedName = DOMUtils::coerceName( $name ); + if ( $name !== $coercedName ) { + $this->coerced = true; + } + return $coercedName; + } + private function createNode( Element $element ) { - $node = $this->doc->createElementNS( - $element->namespace, - $element->name ); + try { + $node = $this->doc->createElementNS( + $element->namespace, + $element->name ); + } catch ( \DOMException $e ) { + // Attempt to escape the name so that it is more acceptable + $node = $this->doc->createElementNS( + $element->namespace, + $this->coerceName( $element->name ) ); + } foreach ( $element->attrs->getObjects() as $attr ) { if ( $attr->namespaceURI === null @@ -98,12 +131,26 @@ // way can't be discovered via hasAttribute() or hasAttributeNS(). $attrNode = $this->doc->createAttribute( $attr->localName ); $attrNode->value = $attr->value; - $node->setAttributeNodeNS( $attrNode ); + try { + $node->setAttributeNodeNS( $attrNode ); + } catch ( \DOMException $e ) { + $node->setAttributeNS( + $attr->namespaceURI, + $this->coerceName( $attr->qualifiedName ), + $attr->value ); + } } else { - $node->setAttributeNS( - $attr->namespaceURI, - $attr->qualifiedName, - $attr->value ); + try { + $node->setAttributeNS( + $attr->namespaceURI, + $attr->qualifiedName, + $attr->value ); + } catch ( \DOMException $e ) { + $node->setAttributeNS( + $attr->namespaceURI, + $this->coerceName( $attr->qualifiedName ), + $attr->value ); + } } } $element->userData = $node; @@ -164,17 +211,41 @@ // instead. $attrNode = $this->doc->createAttribute( $attr->localName ); $attrNode->value = $attr->value; - $replaced = $node->setAttributeNodeNS( $attrNode ); + try { + $replaced = $node->setAttributeNodeNS( $attrNode ); + } catch ( \DOMException $e ) { + $attrNode = $this->doc->createAttribute( + $this->coerceName( $attr->localName ) ); + $attrNode->value = $attr->value; + $replaced = $node->setAttributeNodeNS( $attrNode ); + } if ( $replaced ) { // Put it back how it was $node->setAttributeNodeNS( $replaced ); } } elseif ( $attr->namespaceURI === null ) { - if ( !$node->hasAttribute( $attr->localName ) ) { - $node->setAttribute( $attr->localName, $attr->value ); + try { + if ( !$node->hasAttribute( $attr->localName ) ) { + $node->setAttribute( $attr->localName, $attr->value ); + } + } catch ( \DOMException $e ) { + $name = $this->coerceName( $attr->localName ); + if ( !$node->hasAttribute( $name ) ) { + $node->setAttribute( $name, $attr->value ); + } } - } elseif ( !$node->hasAttributeNS( $attr->namespaceURI, $attr->localName ) ) { - $node->setAttributeNS( $attr->namespaceURI, $attr->localName, $attr->value ); + } else { + try { + if ( !$node->hasAttributeNS( $attr->namespaceURI, $attr->localName ) ) { + $node->setAttributeNS( $attr->namespaceURI, + $attr->localName, $attr->value ); + } + } catch ( \DOMException $e ) { + $name = $this->coerceName( $attr->localName ); + if ( !$node->hasAttributeNS( $attr->namespaceURI, $name ) ) { + $node->setAttributeNS( $attr->namespaceURI, $name, $attr->value ); + } + } } } } diff --git a/src/GenerateDataFiles.php b/src/GenerateDataFiles.php index d849596..13430ff 100644 --- a/src/GenerateDataFiles.php +++ b/src/GenerateDataFiles.php @@ -134,6 +134,18 @@ self::NS_SVG => 'foreignObject, desc, title', ]; + // @codingStandardsIgnoreStart + /** + * The NameStartChar production from XML 1.0, but with colon excluded since + * there's a lot of ways to break namespace validation, and we actually need + * this for local names + */ + private static $nameStartChar = '[A-Z] | "_" | [a-z] | [#xC0-#xD6] | [#xD8-#xF6] | [#xF8-#x2FF] | [#x370-#x37D] | [#x37F-#x1FFF] | [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] | [#x3001-#xD7FF] | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] | [#x10000-#xEFFFF]'; + + /** The NameChar production from XML 1.0 */ + private static $nameChar = 'NameStartChar | "-" | "." | [0-9] | #xB7 | [#x0300-#x036F] | [#x203F-#x2040]'; + // @codingStandardsIgnoreEnd + private function makeRegexAlternation( $array ) { $regex = ''; foreach ( $array as $value ) { @@ -143,6 +155,79 @@ $regex .= "\n\t\t" . preg_quote( substr( $value, 1 ), '~' ); } return $regex; + } + + private function getCharRanges( $input, $nonterminals = [] ) { + $ranges = []; + + foreach ( preg_split( '/\s*\|\s*/', $input ) as $case ) { + if ( preg_match( '/^"(.)"$/', $case, $m ) ) { + // Single ASCII character + $ranges[] = [ ord( $m[1] ), ord( $m[1] ) ]; + } elseif ( preg_match( '/^\[(.)-(.)\]$/', $case, $m ) ) { + // ASCII range + $ranges[] = [ ord( $m[1] ), ord( $m[2] ) ]; + } elseif ( preg_match( '/^#x([0-9A-F]+)$/', $case, $m ) ) { + // Single encoded character + $codepoint = intval( $m[1], 16 ); + $ranges[] = [ $codepoint, $codepoint ]; + } elseif ( preg_match( '/^\[#x([0-9A-F]+)-#x([0-9A-F]+)\]$/', $case, $m ) ) { + // Encoded range + $ranges[] = [ intval( $m[1], 16 ), intval( $m[2], 16 ) ]; + } elseif ( isset( $nonterminals[$case] ) ) { + $ranges = array_merge( $ranges, $this->getCharRanges( $nonterminals[$case] ) ); + } else { + throw new \Exception( "Invalid XML char case \"$case\"" ); + } + } + usort( $ranges, function ( $a, $b ) { + return $a[0] - $b[0]; + } ); + return $ranges; + } + + private function makeConvTable( $input, $nonterminals = [] ) { + $ranges = $this->getCharRanges( $input, $nonterminals ); + + // Invert the ranges, produce a set complement + $lastEndPlusOne = 0; + $table = []; + for ( $i = 0; $i < count( $ranges ); $i++ ) { + $start = $ranges[$i][0]; + $end = $ranges[$i][1]; + // Merge consecutive ranges + for ( $j = $i + 1; $j < count( $ranges ); $j++ ) { + if ( $ranges[$j][0] === $end + 1 ) { + $end = $ranges[$j][1]; + $i = $j; + } else { + break; + } + } + + $table[] = $lastEndPlusOne; + $table[] = $start - 1; + $table[] = 0; + $table[] = 0xffffff; + + $lastEndPlusOne = $end + 1; + } + + // Last range + $table[] = $lastEndPlusOne; + $table[] = 0x10ffff; + $table[] = 0; + $table[] = 0xffffff; + + return $table; + } + + private function encodeConvTable( $table ) { + return "[\n\t\t" . implode( ",\n\t\t", array_map( + function ( $a ) { + return implode( ', ', $a ); + }, + array_chunk( $table, 4 ) ) ) . ' ]'; } private function execute() { @@ -188,10 +273,16 @@ $this->makeRegexAlternation( self::$quirkyPublicPrefixes ) . '~xAi'; + $nameStartCharConvTable = $this->makeConvTable( self::$nameStartChar ); + $nameCharConvTable = $this->makeConvTable( self::$nameChar, + [ 'NameStartChar' => self::$nameStartChar ] ); + $encEntityRegex = var_export( $entityRegex, true ); $encTranslations = var_export( $entityTranslations, true ); $encLegacy = var_export( $legacyNumericEntities, true ); $encQuirkyRegex = var_export( $quirkyRegex, true ); + $encNameStartCharConvTable = $this->encodeConvTable( $nameStartCharConvTable ); + $encNameCharConvTable = $this->encodeConvTable( $nameCharConvTable ); $special = []; foreach ( self::$special as $ns => $str ) { @@ -223,6 +314,8 @@ static public \$namedEntityTranslations = $encTranslations; static public \$legacyNumericEntities = $encLegacy; static public \$quirkyPrefixRegex = $encQuirkyRegex; + static public \$nameStartCharConvTable = $encNameStartCharConvTable; + static public \$nameCharConvTable = $encNameCharConvTable; } PHP; diff --git a/src/HTMLData.php b/src/HTMLData.php index 4a47d63..3da2890 100644 --- a/src/HTMLData.php +++ b/src/HTMLData.php @@ -4667,4 +4667,41 @@ //W3O//DTD W3 HTML 3\\.0//| //WebTechs//DTD Mozilla HTML 2\\.0//| //WebTechs//DTD Mozilla HTML//~xAi'; + static public $nameStartCharConvTable = [ + 0, 64, 0, 16777215, + 91, 94, 0, 16777215, + 96, 96, 0, 16777215, + 123, 191, 0, 16777215, + 215, 215, 0, 16777215, + 247, 247, 0, 16777215, + 768, 879, 0, 16777215, + 894, 894, 0, 16777215, + 8192, 8203, 0, 16777215, + 8206, 8303, 0, 16777215, + 8592, 11263, 0, 16777215, + 12272, 12288, 0, 16777215, + 55296, 63743, 0, 16777215, + 64976, 65007, 0, 16777215, + 65534, 65535, 0, 16777215, + 983040, 1114111, 0, 16777215 ]; + static public $nameCharConvTable = [ + 0, 44, 0, 16777215, + 47, 47, 0, 16777215, + 58, 64, 0, 16777215, + 91, 94, 0, 16777215, + 96, 96, 0, 16777215, + 123, 182, 0, 16777215, + 184, 191, 0, 16777215, + 215, 215, 0, 16777215, + 247, 247, 0, 16777215, + 894, 894, 0, 16777215, + 8192, 8203, 0, 16777215, + 8206, 8254, 0, 16777215, + 8257, 8303, 0, 16777215, + 8592, 11263, 0, 16777215, + 12272, 12288, 0, 16777215, + 55296, 63743, 0, 16777215, + 64976, 65007, 0, 16777215, + 65534, 65535, 0, 16777215, + 983040, 1114111, 0, 16777215 ]; } \ No newline at end of file diff --git a/src/Serializer/HtmlFormatter.php b/src/Serializer/HtmlFormatter.php index db61857..5445c1e 100644 --- a/src/Serializer/HtmlFormatter.php +++ b/src/Serializer/HtmlFormatter.php @@ -2,6 +2,7 @@ namespace RemexHtml\Serializer; use RemexHtml\HTMLData; +use RemexHtml\DOM\DOMUtils; use RemexHtml\DOM\DOMFormatter; /** @@ -83,6 +84,7 @@ ]; protected $useSourceDoctype; + protected $reverseCoercion; /** * Constructor. @@ -91,16 +93,20 @@ * - scriptingFlag : Set this to false to disable scripting. True by default. * - useSourceDoctype : Emit the doctype used in the source. If this is * false or absent, an HTML doctype will be used. + * - reverseCoercion : When formatting a DOM node, reverse the encoding + * of invalid names. False by default. */ public function __construct( $options = [] ) { $options += [ 'scriptingFlag' => true, 'useSourceDoctype' => false, + 'reverseCoercion' => false, ]; if ( $options['scriptingFlag'] ) { $this->rawTextElements['noscript'] = true; } $this->useSourceDoctype = $options['useSourceDoctype']; + $this->reverseCoercion = $options['reverseCoercion']; } public function startDocument( $fragmentNamespace, $fragmentName ) { @@ -216,6 +222,10 @@ } else { $name = $node->prefix . ':' . $node->localName; } + if ( $this->reverseCoercion ) { + $name = DOMUtils::uncoerceName( $name ); + } + $s = '<' . $name; foreach ( $node->attributes as $attr ) { switch ( $attr->namespaceURI ) { @@ -239,6 +249,9 @@ $attrName = $attr->localName; } } + if ( $this->reverseCoercion ) { + $attrName = DOMUtils::uncoerceName( $attrName ); + } $encValue = strtr( $attr->value, $this->attributeEscapes ); $s .= " $attrName=\"$encValue\""; } diff --git a/src/Serializer/TestFormatter.php b/src/Serializer/TestFormatter.php index 65af2b3..b2580b2 100644 --- a/src/Serializer/TestFormatter.php +++ b/src/Serializer/TestFormatter.php @@ -6,6 +6,7 @@ use RemexHtml\Tokenizer\PlainAttributes; use RemexHtml\HTMLData; use RemexHtml\DOM\DOMFormatter; +use RemexHtml\DOM\DOMUtils; /** * A Formatter which is used to format documents in (almost) the way they @@ -48,6 +49,7 @@ } private function formatElement( $namespace, $name, $attrs, $contents ) { + $name = DOMUtils::uncoerceName( $name ); if ( $namespace === HTMLData::NS_HTML ) { $tagName = $name; } elseif ( $namespace === HTMLData::NS_SVG ) { @@ -61,6 +63,7 @@ $sortedAttrs = $attrs; ksort( $sortedAttrs, SORT_STRING ); foreach ( $sortedAttrs as $attrName => $attr ) { + $localName = DOMUtils::uncoerceName( $attr->localName ); if ( $attr->namespaceURI === null || isset( $attr->reallyNoNamespace ) ) { @@ -68,7 +71,7 @@ } elseif ( isset( self::$attrNamespaces[$attr->namespaceURI] ) ) { $prefix = self::$attrNamespaces[$attr->namespaceURI] . ' '; } - $ret .= " $prefix{$attr->localName}=\"{$attr->value}\"\n"; + $ret .= " $prefix$localName=\"{$attr->value}\"\n"; } if ( $contents !== null && $contents !== '' ) { $contents = preg_replace( '/^/m', ' ', $contents ); diff --git a/tests/phpunit/TreeBuilderTest.php b/tests/phpunit/TreeBuilderTest.php index 63fbf6f..b9aeeeb 100644 --- a/tests/phpunit/TreeBuilderTest.php +++ b/tests/phpunit/TreeBuilderTest.php @@ -21,19 +21,6 @@ ]; private static $domTestBlacklist = [ - // Invalid tag name - 'tree-construction/html5test-com.dat:1', - 'tree-construction/webkit01.dat:179', - - // Invalid attribute name - 'tree-construction/html5test-com.dat:12', - 'tree-construction/html5test-com.dat:39', - 'tree-construction/tests14.dat:45', - 'tree-construction/tests14.dat:55', - 'tree-construction/tests14.dat:67', - 'tree-construction/tests26.dat:263', - 'tree-construction/webkit01.dat:606', - // Invalid doctype 'tree-construction/doctype01.dat:32', 'tree-construction/doctype01.dat:45', -- To view, visit https://gerrit.wikimedia.org/r/335596 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Iddfa497e07636693baa280575b086f46adefe002 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/libs/RemexHtml Gerrit-Branch: master Gerrit-Owner: Tim Starling <tstarl...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits