Author: kn Date: Fri Feb 8 15:21:30 2008 New Revision: 7326 Log: - Added support for typical text inline markup tokens
Added: experimental/Document/tests/files/rst/tokenizer/paragraph.tokens Modified: experimental/Document/src/document/rst/token.php experimental/Document/src/document/rst/tokenizer.php experimental/Document/tests/files/rst/tokenizer/bullet_list.tokens experimental/Document/tests/files/rst/tokenizer/definition_list.tokens experimental/Document/tests/files/rst/tokenizer/titles.tokens Modified: experimental/Document/src/document/rst/token.php ============================================================================== --- experimental/Document/src/document/rst/token.php [iso-8859-1] (original) +++ experimental/Document/src/document/rst/token.php [iso-8859-1] Fri Feb 8 15:21:30 2008 @@ -25,6 +25,17 @@ const HEADLINE = 11; const BULLET_POINT = 21; + + const QUOTE = 50; + const SINGLE_QUOTE = 51; + const DOUBLE_QUOTE = 52; + const ASTERISK = 53; + const UNDERSCORE = 54; + const ROUND_BRACKET_OPEN = 55; + const ROUND_BRACKET_CLOSE = 56; + const SQUARE_BRACKET_OPEN = 57; + const SQUARE_BRACKET_CLOSE = 58; + const PIPE = 59; const TEXT_LINE = 99; Modified: experimental/Document/src/document/rst/tokenizer.php ============================================================================== --- experimental/Document/src/document/rst/tokenizer.php [iso-8859-1] (original) +++ experimental/Document/src/document/rst/tokenizer.php [iso-8859-1] Fri Feb 8 15:21:30 2008 @@ -59,7 +59,7 @@ * * @see http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#enumerated-lists */ - const TEXT_END_CHARS = '`\'\\s'; + const TEXT_END_CHARS = '`*_[\\]|()"\'\r\n'; /** * List with tokens and a regular expression matching the given token. @@ -81,20 +81,45 @@ public function __construct() { $this->tokens = array( + // Whitespaces ezcDocumentRstToken::WHITESPACE => '(\\A(?P<value>[' . self::WHITESPACE_CHARS . ']+))', ezcDocumentRstToken::NEWLINE => '(\\A(?P<value>\\r\\n|\\r|\\n))', + // Document structure ezcDocumentRstToken::HEADLINE => '(\\A(?P<value>([' . self::HEADLINE_CHARS . '])\\2*)$)m', + // Lists ezcDocumentRstToken::BULLET_POINT => '(\\A(?P<value>[' . self::BULLET_LIST_CHARS . '][' . self::WHITESPACE_CHARS . ']))u', + // Special characters in text + ezcDocumentRstToken::QUOTE => + '(\\A(?P<value>`))', + ezcDocumentRstToken::SINGLE_QUOTE => + '(\\A(?P<value>\'))', + ezcDocumentRstToken::DOUBLE_QUOTE => + '(\\A(?P<value>\"))', + ezcDocumentRstToken::ASTERISK => + '(\\A(?P<value>\\*))', + ezcDocumentRstToken::UNDERSCORE => + '(\\A(?P<value>_))', + ezcDocumentRstToken::ROUND_BRACKET_OPEN => + '(\\A(?P<value>\\())', + ezcDocumentRstToken::ROUND_BRACKET_CLOSE => + '(\\A(?P<value>\\)))', + ezcDocumentRstToken::SQUARE_BRACKET_OPEN => + '(\\A(?P<value>\\[))', + ezcDocumentRstToken::SQUARE_BRACKET_CLOSE => + '(\\A(?P<value>]))', + ezcDocumentRstToken::PIPE => + '(\\A(?P<value>\\|))', + // This should be last match ezcDocumentRstToken::TEXT_LINE => - '(\\A(?P<value>[^' . self::TEXT_END_CHARS . ']*))', + '(\\A(?P<value>[^' . self::TEXT_END_CHARS . ']+))', ); } Modified: experimental/Document/tests/files/rst/tokenizer/bullet_list.tokens ============================================================================== --- experimental/Document/tests/files/rst/tokenizer/bullet_list.tokens [iso-8859-1] (original) +++ experimental/Document/tests/files/rst/tokenizer/bullet_list.tokens [iso-8859-1] Fri Feb 8 15:21:30 2008 @@ -11,67 +11,11 @@ 1 => ezcDocumentRstToken::__set_state(array( 'type' => 99, - 'content' => 'This', + 'content' => 'This is a bullet list.', 'line' => 1, 'position' => 3, )), 2 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 1, - 'position' => 7, - )), - 3 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'is', - 'line' => 1, - 'position' => 8, - )), - 4 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 1, - 'position' => 10, - )), - 5 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'a', - 'line' => 1, - 'position' => 11, - )), - 6 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 1, - 'position' => 12, - )), - 7 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'bullet', - 'line' => 1, - 'position' => 13, - )), - 8 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 1, - 'position' => 19, - )), - 9 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'list.', - 'line' => 1, - 'position' => 20, - )), - 10 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -79,7 +23,7 @@ 'line' => 1, 'position' => 25, )), - 11 => + 3 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -87,105 +31,105 @@ 'line' => 2, 'position' => 1, )), - 12 => + 4 => ezcDocumentRstToken::__set_state(array( 'type' => 21, 'content' => '- ', 'line' => 3, 'position' => 1, )), + 5 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'Bullets can be ', + 'line' => 3, + 'position' => 3, + )), + 6 => + ezcDocumentRstToken::__set_state(array( + 'type' => 52, + 'content' => '"', + 'line' => 3, + 'position' => 18, + )), + 7 => + ezcDocumentRstToken::__set_state(array( + 'type' => 53, + 'content' => '*', + 'line' => 3, + 'position' => 19, + )), + 8 => + ezcDocumentRstToken::__set_state(array( + 'type' => 52, + 'content' => '"', + 'line' => 3, + 'position' => 20, + )), + 9 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => ', ', + 'line' => 3, + 'position' => 21, + )), + 10 => + ezcDocumentRstToken::__set_state(array( + 'type' => 52, + 'content' => '"', + 'line' => 3, + 'position' => 23, + )), + 11 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => '+', + 'line' => 3, + 'position' => 24, + )), + 12 => + ezcDocumentRstToken::__set_state(array( + 'type' => 52, + 'content' => '"', + 'line' => 3, + 'position' => 25, + )), 13 => ezcDocumentRstToken::__set_state(array( 'type' => 99, - 'content' => 'Bullets', + 'content' => ', or ', 'line' => 3, - 'position' => 3, + 'position' => 26, )), 14 => ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', + 'type' => 52, + 'content' => '"', 'line' => 3, - 'position' => 10, + 'position' => 31, )), 15 => ezcDocumentRstToken::__set_state(array( 'type' => 99, - 'content' => 'can', + 'content' => '-', 'line' => 3, - 'position' => 11, + 'position' => 32, )), 16 => ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', + 'type' => 52, + 'content' => '"', 'line' => 3, - 'position' => 14, + 'position' => 33, )), 17 => ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'be', + 'type' => 11, + 'content' => '.', 'line' => 3, - 'position' => 15, + 'position' => 34, )), 18 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 3, - 'position' => 17, - )), - 19 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => '"*",', - 'line' => 3, - 'position' => 18, - )), - 20 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 3, - 'position' => 22, - )), - 21 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => '"+",', - 'line' => 3, - 'position' => 23, - )), - 22 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 3, - 'position' => 27, - )), - 23 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'or', - 'line' => 3, - 'position' => 28, - )), - 24 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 3, - 'position' => 30, - )), - 25 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => '"-".', - 'line' => 3, - 'position' => 31, - )), - 26 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' Modified: experimental/Document/tests/files/rst/tokenizer/definition_list.tokens ============================================================================== --- experimental/Document/tests/files/rst/tokenizer/definition_list.tokens [iso-8859-1] (original) +++ experimental/Document/tests/files/rst/tokenizer/definition_list.tokens [iso-8859-1] Fri Feb 8 15:21:30 2008 @@ -26,109 +26,11 @@ 3 => ezcDocumentRstToken::__set_state(array( 'type' => 99, - 'content' => 'Definition', + 'content' => 'Definition lists associate a term with a definition.', 'line' => 2, 'position' => 5, )), 4 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 2, - 'position' => 15, - )), - 5 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'lists', - 'line' => 2, - 'position' => 16, - )), - 6 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 2, - 'position' => 21, - )), - 7 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'associate', - 'line' => 2, - 'position' => 22, - )), - 8 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 2, - 'position' => 31, - )), - 9 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'a', - 'line' => 2, - 'position' => 32, - )), - 10 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 2, - 'position' => 33, - )), - 11 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'term', - 'line' => 2, - 'position' => 34, - )), - 12 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 2, - 'position' => 38, - )), - 13 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'with', - 'line' => 2, - 'position' => 39, - )), - 14 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 2, - 'position' => 43, - )), - 15 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'a', - 'line' => 2, - 'position' => 44, - )), - 16 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 2, - 'position' => 45, - )), - 17 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'definition.', - 'line' => 2, - 'position' => 46, - )), - 18 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -136,7 +38,7 @@ 'line' => 2, 'position' => 57, )), - 19 => + 5 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -144,14 +46,14 @@ 'line' => 3, 'position' => 1, )), - 20 => + 6 => ezcDocumentRstToken::__set_state(array( 'type' => 99, 'content' => 'how', 'line' => 4, 'position' => 1, )), - 21 => + 7 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -159,161 +61,21 @@ 'line' => 4, 'position' => 4, )), - 22 => + 8 => ezcDocumentRstToken::__set_state(array( 'type' => 1, 'content' => ' ', 'line' => 5, 'position' => 1, )), - 23 => + 9 => ezcDocumentRstToken::__set_state(array( 'type' => 99, - 'content' => 'The', + 'content' => 'The term is a one-line phrase, and the definition is one', 'line' => 5, 'position' => 5, )), - 24 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 5, - 'position' => 8, - )), - 25 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'term', - 'line' => 5, - 'position' => 9, - )), - 26 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 5, - 'position' => 13, - )), - 27 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'is', - 'line' => 5, - 'position' => 14, - )), - 28 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 5, - 'position' => 16, - )), - 29 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'a', - 'line' => 5, - 'position' => 17, - )), - 30 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 5, - 'position' => 18, - )), - 31 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'one-line', - 'line' => 5, - 'position' => 19, - )), - 32 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 5, - 'position' => 27, - )), - 33 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'phrase,', - 'line' => 5, - 'position' => 28, - )), - 34 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 5, - 'position' => 35, - )), - 35 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'and', - 'line' => 5, - 'position' => 36, - )), - 36 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 5, - 'position' => 39, - )), - 37 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'the', - 'line' => 5, - 'position' => 40, - )), - 38 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 5, - 'position' => 43, - )), - 39 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'definition', - 'line' => 5, - 'position' => 44, - )), - 40 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 5, - 'position' => 54, - )), - 41 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'is', - 'line' => 5, - 'position' => 55, - )), - 42 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 5, - 'position' => 57, - )), - 43 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'one', - 'line' => 5, - 'position' => 58, - )), - 44 => + 10 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -321,133 +83,21 @@ 'line' => 5, 'position' => 61, )), - 45 => + 11 => ezcDocumentRstToken::__set_state(array( 'type' => 1, 'content' => ' ', 'line' => 6, 'position' => 1, )), - 46 => + 12 => ezcDocumentRstToken::__set_state(array( 'type' => 99, - 'content' => 'or', + 'content' => 'or more paragraphs or body elements, indented relative to', 'line' => 6, 'position' => 5, )), - 47 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 6, - 'position' => 7, - )), - 48 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'more', - 'line' => 6, - 'position' => 8, - )), - 49 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 6, - 'position' => 12, - )), - 50 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'paragraphs', - 'line' => 6, - 'position' => 13, - )), - 51 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 6, - 'position' => 23, - )), - 52 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'or', - 'line' => 6, - 'position' => 24, - )), - 53 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 6, - 'position' => 26, - )), - 54 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'body', - 'line' => 6, - 'position' => 27, - )), - 55 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 6, - 'position' => 31, - )), - 56 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'elements,', - 'line' => 6, - 'position' => 32, - )), - 57 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 6, - 'position' => 41, - )), - 58 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'indented', - 'line' => 6, - 'position' => 42, - )), - 59 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 6, - 'position' => 50, - )), - 60 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'relative', - 'line' => 6, - 'position' => 51, - )), - 61 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 6, - 'position' => 59, - )), - 62 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'to', - 'line' => 6, - 'position' => 60, - )), - 63 => + 13 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -455,35 +105,21 @@ 'line' => 6, 'position' => 62, )), - 64 => + 14 => ezcDocumentRstToken::__set_state(array( 'type' => 1, 'content' => ' ', 'line' => 7, 'position' => 1, )), - 65 => + 15 => ezcDocumentRstToken::__set_state(array( 'type' => 99, - 'content' => 'the', + 'content' => 'the term.', 'line' => 7, 'position' => 5, )), - 66 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 7, - 'position' => 8, - )), - 67 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'term.', - 'line' => 7, - 'position' => 9, - )), - 68 => + 16 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' Added: experimental/Document/tests/files/rst/tokenizer/paragraph.tokens ============================================================================== --- experimental/Document/tests/files/rst/tokenizer/paragraph.tokens (added) +++ experimental/Document/tests/files/rst/tokenizer/paragraph.tokens [iso-8859-1] Fri Feb 8 15:21:30 2008 @@ -1,0 +1,525 @@ +<?php + +return array ( + 0 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'Paragraphs contain text and may contain inline markup:', + 'line' => 1, + 'position' => 1, + )), + 1 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', + 'line' => 1, + 'position' => 55, + )), + 2 => + ezcDocumentRstToken::__set_state(array( + 'type' => 53, + 'content' => '*', + 'line' => 2, + 'position' => 1, + )), + 3 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'emphasis', + 'line' => 2, + 'position' => 2, + )), + 4 => + ezcDocumentRstToken::__set_state(array( + 'type' => 53, + 'content' => '*', + 'line' => 2, + 'position' => 10, + )), + 5 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => ', ', + 'line' => 2, + 'position' => 11, + )), + 6 => + ezcDocumentRstToken::__set_state(array( + 'type' => 53, + 'content' => '*', + 'line' => 2, + 'position' => 13, + )), + 7 => + ezcDocumentRstToken::__set_state(array( + 'type' => 53, + 'content' => '*', + 'line' => 2, + 'position' => 14, + )), + 8 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'strong emphasis', + 'line' => 2, + 'position' => 15, + )), + 9 => + ezcDocumentRstToken::__set_state(array( + 'type' => 53, + 'content' => '*', + 'line' => 2, + 'position' => 30, + )), + 10 => + ezcDocumentRstToken::__set_state(array( + 'type' => 53, + 'content' => '*', + 'line' => 2, + 'position' => 31, + )), + 11 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => ', ', + 'line' => 2, + 'position' => 32, + )), + 12 => + ezcDocumentRstToken::__set_state(array( + 'type' => 50, + 'content' => '`', + 'line' => 2, + 'position' => 34, + )), + 13 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'interpreted text', + 'line' => 2, + 'position' => 35, + )), + 14 => + ezcDocumentRstToken::__set_state(array( + 'type' => 50, + 'content' => '`', + 'line' => 2, + 'position' => 51, + )), + 15 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => ', ', + 'line' => 2, + 'position' => 52, + )), + 16 => + ezcDocumentRstToken::__set_state(array( + 'type' => 50, + 'content' => '`', + 'line' => 2, + 'position' => 54, + )), + 17 => + ezcDocumentRstToken::__set_state(array( + 'type' => 50, + 'content' => '`', + 'line' => 2, + 'position' => 55, + )), + 18 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'inline', + 'line' => 2, + 'position' => 56, + )), + 19 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', + 'line' => 2, + 'position' => 62, + )), + 20 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'literals', + 'line' => 3, + 'position' => 1, + )), + 21 => + ezcDocumentRstToken::__set_state(array( + 'type' => 50, + 'content' => '`', + 'line' => 3, + 'position' => 9, + )), + 22 => + ezcDocumentRstToken::__set_state(array( + 'type' => 50, + 'content' => '`', + 'line' => 3, + 'position' => 10, + )), + 23 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => ', standalone hyperlinks ', + 'line' => 3, + 'position' => 11, + )), + 24 => + ezcDocumentRstToken::__set_state(array( + 'type' => 55, + 'content' => '(', + 'line' => 3, + 'position' => 35, + )), + 25 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'http://www.python.org', + 'line' => 3, + 'position' => 36, + )), + 26 => + ezcDocumentRstToken::__set_state(array( + 'type' => 56, + 'content' => ')', + 'line' => 3, + 'position' => 57, + )), + 27 => + ezcDocumentRstToken::__set_state(array( + 'type' => 11, + 'content' => ',', + 'line' => 3, + 'position' => 58, + )), + 28 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', + 'line' => 3, + 'position' => 59, + )), + 29 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'external hyperlinks ', + 'line' => 4, + 'position' => 1, + )), + 30 => + ezcDocumentRstToken::__set_state(array( + 'type' => 55, + 'content' => '(', + 'line' => 4, + 'position' => 21, + )), + 31 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'Python', + 'line' => 4, + 'position' => 22, + )), + 32 => + ezcDocumentRstToken::__set_state(array( + 'type' => 54, + 'content' => '_', + 'line' => 4, + 'position' => 28, + )), + 33 => + ezcDocumentRstToken::__set_state(array( + 'type' => 56, + 'content' => ')', + 'line' => 4, + 'position' => 29, + )), + 34 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => ', internal cross-references', + 'line' => 4, + 'position' => 30, + )), + 35 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', + 'line' => 4, + 'position' => 57, + )), + 36 => + ezcDocumentRstToken::__set_state(array( + 'type' => 55, + 'content' => '(', + 'line' => 5, + 'position' => 1, + )), + 37 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'example', + 'line' => 5, + 'position' => 2, + )), + 38 => + ezcDocumentRstToken::__set_state(array( + 'type' => 54, + 'content' => '_', + 'line' => 5, + 'position' => 9, + )), + 39 => + ezcDocumentRstToken::__set_state(array( + 'type' => 56, + 'content' => ')', + 'line' => 5, + 'position' => 10, + )), + 40 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => ', footnote references ', + 'line' => 5, + 'position' => 11, + )), + 41 => + ezcDocumentRstToken::__set_state(array( + 'type' => 55, + 'content' => '(', + 'line' => 5, + 'position' => 33, + )), + 42 => + ezcDocumentRstToken::__set_state(array( + 'type' => 57, + 'content' => '[', + 'line' => 5, + 'position' => 34, + )), + 43 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => '1', + 'line' => 5, + 'position' => 35, + )), + 44 => + ezcDocumentRstToken::__set_state(array( + 'type' => 58, + 'content' => ']', + 'line' => 5, + 'position' => 36, + )), + 45 => + ezcDocumentRstToken::__set_state(array( + 'type' => 54, + 'content' => '_', + 'line' => 5, + 'position' => 37, + )), + 46 => + ezcDocumentRstToken::__set_state(array( + 'type' => 56, + 'content' => ')', + 'line' => 5, + 'position' => 38, + )), + 47 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => ', citation references', + 'line' => 5, + 'position' => 39, + )), + 48 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', + 'line' => 5, + 'position' => 60, + )), + 49 => + ezcDocumentRstToken::__set_state(array( + 'type' => 55, + 'content' => '(', + 'line' => 6, + 'position' => 1, + )), + 50 => + ezcDocumentRstToken::__set_state(array( + 'type' => 57, + 'content' => '[', + 'line' => 6, + 'position' => 2, + )), + 51 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'CIT2002', + 'line' => 6, + 'position' => 3, + )), + 52 => + ezcDocumentRstToken::__set_state(array( + 'type' => 58, + 'content' => ']', + 'line' => 6, + 'position' => 10, + )), + 53 => + ezcDocumentRstToken::__set_state(array( + 'type' => 54, + 'content' => '_', + 'line' => 6, + 'position' => 11, + )), + 54 => + ezcDocumentRstToken::__set_state(array( + 'type' => 56, + 'content' => ')', + 'line' => 6, + 'position' => 12, + )), + 55 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => ', substitution references ', + 'line' => 6, + 'position' => 13, + )), + 56 => + ezcDocumentRstToken::__set_state(array( + 'type' => 55, + 'content' => '(', + 'line' => 6, + 'position' => 39, + )), + 57 => + ezcDocumentRstToken::__set_state(array( + 'type' => 59, + 'content' => '|', + 'line' => 6, + 'position' => 40, + )), + 58 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'example', + 'line' => 6, + 'position' => 41, + )), + 59 => + ezcDocumentRstToken::__set_state(array( + 'type' => 59, + 'content' => '|', + 'line' => 6, + 'position' => 48, + )), + 60 => + ezcDocumentRstToken::__set_state(array( + 'type' => 56, + 'content' => ')', + 'line' => 6, + 'position' => 49, + )), + 61 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => ', and ', + 'line' => 6, + 'position' => 50, + )), + 62 => + ezcDocumentRstToken::__set_state(array( + 'type' => 54, + 'content' => '_', + 'line' => 6, + 'position' => 56, + )), + 63 => + ezcDocumentRstToken::__set_state(array( + 'type' => 50, + 'content' => '`', + 'line' => 6, + 'position' => 57, + )), + 64 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'inline', + 'line' => 6, + 'position' => 58, + )), + 65 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', + 'line' => 6, + 'position' => 64, + )), + 66 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'internal targets', + 'line' => 7, + 'position' => 1, + )), + 67 => + ezcDocumentRstToken::__set_state(array( + 'type' => 50, + 'content' => '`', + 'line' => 7, + 'position' => 17, + )), + 68 => + ezcDocumentRstToken::__set_state(array( + 'type' => 11, + 'content' => '.', + 'line' => 7, + 'position' => 18, + )), + 69 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', + 'line' => 7, + 'position' => 19, + )), + 70 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', + 'line' => 8, + 'position' => 1, + )), + 71 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'Paragraphs are separated by blank lines and are left-aligned.', + 'line' => 9, + 'position' => 1, + )), + 72 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', + 'line' => 9, + 'position' => 62, + )), +); + Modified: experimental/Document/tests/files/rst/tokenizer/titles.tokens ============================================================================== --- experimental/Document/tests/files/rst/tokenizer/titles.tokens [iso-8859-1] (original) +++ experimental/Document/tests/files/rst/tokenizer/titles.tokens [iso-8859-1] Fri Feb 8 15:21:30 2008 @@ -26,40 +26,26 @@ 3 => ezcDocumentRstToken::__set_state(array( 'type' => 99, - 'content' => 'Section', + 'content' => 'Section Title', 'line' => 2, 'position' => 2, )), 4 => ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 2, - 'position' => 9, - )), - 5 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Title', - 'line' => 2, - 'position' => 10, - )), - 6 => - ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' ', 'line' => 2, 'position' => 15, )), - 7 => + 5 => ezcDocumentRstToken::__set_state(array( 'type' => 11, 'content' => '===============', 'line' => 3, 'position' => 1, )), - 8 => + 6 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -67,7 +53,7 @@ 'line' => 3, 'position' => 16, )), - 9 => + 7 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -75,14 +61,14 @@ 'line' => 4, 'position' => 1, )), - 10 => + 8 => ezcDocumentRstToken::__set_state(array( 'type' => 11, 'content' => '---------------', 'line' => 5, 'position' => 1, )), - 11 => + 9 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -90,35 +76,21 @@ 'line' => 5, 'position' => 16, )), - 12 => + 10 => ezcDocumentRstToken::__set_state(array( 'type' => 1, 'content' => ' ', 'line' => 6, 'position' => 1, )), - 13 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Section', + 11 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'Section Title', 'line' => 6, 'position' => 2, )), - 14 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', - 'line' => 6, - 'position' => 9, - )), - 15 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Title', - 'line' => 6, - 'position' => 10, - )), - 16 => + 12 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -126,14 +98,14 @@ 'line' => 6, 'position' => 15, )), - 17 => + 13 => ezcDocumentRstToken::__set_state(array( 'type' => 11, 'content' => '---------------', 'line' => 7, 'position' => 1, )), - 18 => + 14 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -141,7 +113,7 @@ 'line' => 7, 'position' => 16, )), - 19 => + 15 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -149,43 +121,29 @@ 'line' => 8, 'position' => 1, )), - 20 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Section', + 16 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'Section Title', 'line' => 9, 'position' => 1, )), - 21 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', + 17 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', 'line' => 9, - 'position' => 8, - )), - 22 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Title', - 'line' => 9, - 'position' => 9, - )), - 23 => - ezcDocumentRstToken::__set_state(array( - 'type' => 2, - 'content' => ' -', - 'line' => 9, - 'position' => 14, - )), - 24 => + 'position' => 14, + )), + 18 => ezcDocumentRstToken::__set_state(array( 'type' => 11, 'content' => '=============', 'line' => 10, 'position' => 1, )), - 25 => + 19 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -193,7 +151,7 @@ 'line' => 10, 'position' => 14, )), - 26 => + 20 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -201,43 +159,29 @@ 'line' => 11, 'position' => 1, )), - 27 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Section', + 21 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'Section Title', 'line' => 12, 'position' => 1, )), - 28 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', + 22 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', 'line' => 12, - 'position' => 8, - )), - 29 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Title', - 'line' => 12, - 'position' => 9, - )), - 30 => - ezcDocumentRstToken::__set_state(array( - 'type' => 2, - 'content' => ' -', - 'line' => 12, - 'position' => 14, - )), - 31 => + 'position' => 14, + )), + 23 => ezcDocumentRstToken::__set_state(array( 'type' => 11, 'content' => '-------------', 'line' => 13, 'position' => 1, )), - 32 => + 24 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -245,7 +189,7 @@ 'line' => 13, 'position' => 14, )), - 33 => + 25 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -253,43 +197,29 @@ 'line' => 14, 'position' => 1, )), - 34 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Section', + 26 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'Section Title', 'line' => 15, 'position' => 1, )), - 35 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', + 27 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', 'line' => 15, - 'position' => 8, - )), - 36 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Title', - 'line' => 15, - 'position' => 9, - )), - 37 => - ezcDocumentRstToken::__set_state(array( - 'type' => 2, - 'content' => ' -', - 'line' => 15, - 'position' => 14, - )), - 38 => + 'position' => 14, + )), + 28 => ezcDocumentRstToken::__set_state(array( 'type' => 11, 'content' => '`````````````', 'line' => 16, 'position' => 1, )), - 39 => + 29 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -297,7 +227,7 @@ 'line' => 16, 'position' => 14, )), - 40 => + 30 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -305,43 +235,29 @@ 'line' => 17, 'position' => 1, )), - 41 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Section', + 31 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'Section Title', 'line' => 18, 'position' => 1, )), - 42 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', + 32 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', 'line' => 18, - 'position' => 8, - )), - 43 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Title', - 'line' => 18, - 'position' => 9, - )), - 44 => - ezcDocumentRstToken::__set_state(array( - 'type' => 2, - 'content' => ' -', - 'line' => 18, - 'position' => 14, - )), - 45 => + 'position' => 14, + )), + 33 => ezcDocumentRstToken::__set_state(array( 'type' => 11, 'content' => '\'\'\'\'\'\'\'\'\'\'\'\'\'', 'line' => 19, 'position' => 1, )), - 46 => + 34 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -349,7 +265,7 @@ 'line' => 19, 'position' => 14, )), - 47 => + 35 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -357,43 +273,29 @@ 'line' => 20, 'position' => 1, )), - 48 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Section', + 36 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'Section Title', 'line' => 21, 'position' => 1, )), - 49 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', + 37 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', 'line' => 21, - 'position' => 8, - )), - 50 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Title', - 'line' => 21, - 'position' => 9, - )), - 51 => - ezcDocumentRstToken::__set_state(array( - 'type' => 2, - 'content' => ' -', - 'line' => 21, - 'position' => 14, - )), - 52 => + 'position' => 14, + )), + 38 => ezcDocumentRstToken::__set_state(array( 'type' => 11, 'content' => '.............', 'line' => 22, 'position' => 1, )), - 53 => + 39 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -401,7 +303,7 @@ 'line' => 22, 'position' => 14, )), - 54 => + 40 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -409,43 +311,29 @@ 'line' => 23, 'position' => 1, )), - 55 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Section', + 41 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'Section Title', 'line' => 24, 'position' => 1, )), - 56 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', + 42 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', 'line' => 24, - 'position' => 8, - )), - 57 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Title', - 'line' => 24, - 'position' => 9, - )), - 58 => - ezcDocumentRstToken::__set_state(array( - 'type' => 2, - 'content' => ' -', - 'line' => 24, - 'position' => 14, - )), - 59 => + 'position' => 14, + )), + 43 => ezcDocumentRstToken::__set_state(array( 'type' => 11, 'content' => '~~~~~~~~~~~~~', 'line' => 25, 'position' => 1, )), - 60 => + 44 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -453,7 +341,7 @@ 'line' => 25, 'position' => 14, )), - 61 => + 45 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -461,43 +349,29 @@ 'line' => 26, 'position' => 1, )), - 62 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Section', + 46 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'Section Title', 'line' => 27, 'position' => 1, )), - 63 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', + 47 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', 'line' => 27, - 'position' => 8, - )), - 64 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Title', - 'line' => 27, - 'position' => 9, - )), - 65 => - ezcDocumentRstToken::__set_state(array( - 'type' => 2, - 'content' => ' -', - 'line' => 27, - 'position' => 14, - )), - 66 => + 'position' => 14, + )), + 48 => ezcDocumentRstToken::__set_state(array( 'type' => 11, 'content' => '*************', 'line' => 28, 'position' => 1, )), - 67 => + 49 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -505,7 +379,7 @@ 'line' => 28, 'position' => 14, )), - 68 => + 50 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -513,43 +387,29 @@ 'line' => 29, 'position' => 1, )), - 69 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Section', + 51 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'Section Title', 'line' => 30, 'position' => 1, )), - 70 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', + 52 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', 'line' => 30, - 'position' => 8, - )), - 71 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Title', - 'line' => 30, - 'position' => 9, - )), - 72 => - ezcDocumentRstToken::__set_state(array( - 'type' => 2, - 'content' => ' -', - 'line' => 30, - 'position' => 14, - )), - 73 => + 'position' => 14, + )), + 53 => ezcDocumentRstToken::__set_state(array( 'type' => 11, 'content' => '+++++++++++++', 'line' => 31, 'position' => 1, )), - 74 => + 54 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -557,7 +417,7 @@ 'line' => 31, 'position' => 14, )), - 75 => + 55 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' @@ -565,43 +425,29 @@ 'line' => 32, 'position' => 1, )), - 76 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Section', + 56 => + ezcDocumentRstToken::__set_state(array( + 'type' => 99, + 'content' => 'Section Title', 'line' => 33, 'position' => 1, )), - 77 => - ezcDocumentRstToken::__set_state(array( - 'type' => 1, - 'content' => ' ', + 57 => + ezcDocumentRstToken::__set_state(array( + 'type' => 2, + 'content' => ' +', 'line' => 33, - 'position' => 8, - )), - 78 => - ezcDocumentRstToken::__set_state(array( - 'type' => 99, - 'content' => 'Title', - 'line' => 33, - 'position' => 9, - )), - 79 => - ezcDocumentRstToken::__set_state(array( - 'type' => 2, - 'content' => ' -', - 'line' => 33, - 'position' => 14, - )), - 80 => + 'position' => 14, + )), + 58 => ezcDocumentRstToken::__set_state(array( 'type' => 11, 'content' => '^^^^^^^^^^^^^', 'line' => 34, 'position' => 1, )), - 81 => + 59 => ezcDocumentRstToken::__set_state(array( 'type' => 2, 'content' => ' -- svn-components mailing list svn-components@lists.ez.no http://lists.ez.no/mailman/listinfo/svn-components