Author: kn
Date: Mon Feb 11 11:31:10 2008
New Revision: 7333
Log:
- Do not include tarinling whitespaces in texts
- Split texts on multiple whitespaces
Modified:
experimental/Document/src/document/rst/tokenizer.php
experimental/Document/tests/document_rst_tokenizer_tests.php
experimental/Document/tests/files/rst/tokenizer/additional_references.tokens
experimental/Document/tests/files/rst/tokenizer/blockquote.tokens
experimental/Document/tests/files/rst/tokenizer/blockquote_attribution.tokens
experimental/Document/tests/files/rst/tokenizer/bullet_list.tokens
experimental/Document/tests/files/rst/tokenizer/bullet_list_deep.tokens
experimental/Document/tests/files/rst/tokenizer/bullet_list_incorrect.tokens
experimental/Document/tests/files/rst/tokenizer/citation_full.tokens
experimental/Document/tests/files/rst/tokenizer/comment.tokens
experimental/Document/tests/files/rst/tokenizer/deep_block_quote.tokens
experimental/Document/tests/files/rst/tokenizer/definition_list_alignements.tokens
experimental/Document/tests/files/rst/tokenizer/definition_list_classifier.tokens
experimental/Document/tests/files/rst/tokenizer/embedded_uris.tokens
experimental/Document/tests/files/rst/tokenizer/escaping.tokens
experimental/Document/tests/files/rst/tokenizer/field_list.tokens
experimental/Document/tests/files/rst/tokenizer/footnote_complex.tokens
experimental/Document/tests/files/rst/tokenizer/footnotes.tokens
experimental/Document/tests/files/rst/tokenizer/footnotes_auto.tokens
experimental/Document/tests/files/rst/tokenizer/grid_tables.tokens
experimental/Document/tests/files/rst/tokenizer/grid_tables_complex.tokens
experimental/Document/tests/files/rst/tokenizer/hyperlink_targets.tokens
experimental/Document/tests/files/rst/tokenizer/hyperlink_targets_complex.tokens
experimental/Document/tests/files/rst/tokenizer/hyperlinks.tokens
experimental/Document/tests/files/rst/tokenizer/inline_formatting.tokens
experimental/Document/tests/files/rst/tokenizer/inline_internal_targets.tokens
experimental/Document/tests/files/rst/tokenizer/line_block_indented.tokens
experimental/Document/tests/files/rst/tokenizer/list_quote_paragraph.tokens
experimental/Document/tests/files/rst/tokenizer/literal_block.tokens
experimental/Document/tests/files/rst/tokenizer/literal_block_extra.tokens
experimental/Document/tests/files/rst/tokenizer/literal_block_quoted.tokens
experimental/Document/tests/files/rst/tokenizer/named_reference.tokens
experimental/Document/tests/files/rst/tokenizer/non_aligned_text.tokens
experimental/Document/tests/files/rst/tokenizer/option_list.tokens
experimental/Document/tests/files/rst/tokenizer/option_list_complex.tokens
experimental/Document/tests/files/rst/tokenizer/paragraph.tokens
experimental/Document/tests/files/rst/tokenizer/simple_table.tokens
experimental/Document/tests/files/rst/tokenizer/simple_tables_complex.tokens
experimental/Document/tests/files/rst/tokenizer/substitutions_complex.tokens
Modified: experimental/Document/src/document/rst/tokenizer.php
==============================================================================
--- experimental/Document/src/document/rst/tokenizer.php [iso-8859-1] (original)
+++ experimental/Document/src/document/rst/tokenizer.php [iso-8859-1] Mon Feb
11 11:31:10 2008
@@ -36,7 +36,7 @@
*
* @see
http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#enumerated-lists
*/
- const TEXT_END_CHARS = '`*_\\\\[\\]|()"\':\r\n';
+ const TEXT_END_CHARS = '`*_\\\\[\\]|()"\':\\r\\n\\t ';
/**
* List with tokens and a regular expression matching the given token.
@@ -72,7 +72,7 @@
// This should be last match
ezcDocumentRstToken::TEXT_LINE =>
- '(\\A(?P<value>[^' . self::TEXT_END_CHARS . ']+))',
+ '(\\A(?P<value>(?: [^' . self::TEXT_END_CHARS . ']|[^' .
self::TEXT_END_CHARS . '])+))',
);
}
Modified: experimental/Document/tests/document_rst_tokenizer_tests.php
==============================================================================
--- experimental/Document/tests/document_rst_tokenizer_tests.php [iso-8859-1]
(original)
+++ experimental/Document/tests/document_rst_tokenizer_tests.php [iso-8859-1]
Mon Feb 11 11:31:10 2008
@@ -61,7 +61,7 @@
// Store test file, to have something to compare on failure
$tempDir = $this->createTempDir( 'rst_tokenizer' ) . '/';
- file_put_contents( basename( $to ), "<?php\n\nreturn " . var_export(
$tokens, true ) . ";\n\n" );
+ file_put_contents( $tempDir . basename( $to ), "<?php\n\nreturn " .
var_export( $tokens, true ) . ";\n\n" );
$this->assertEquals(
$expected,
Modified:
experimental/Document/tests/files/rst/tokenizer/additional_references.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/blockquote.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/blockquote_attribution.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/bullet_list.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/bullet_list_deep.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/bullet_list_incorrect.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/citation_full.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/comment.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/deep_block_quote.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/definition_list_alignements.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/definition_list_classifier.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/embedded_uris.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/escaping.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/field_list.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/footnote_complex.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/footnotes.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/footnotes_auto.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/grid_tables.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/grid_tables_complex.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/hyperlink_targets.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/hyperlink_targets_complex.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/hyperlinks.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/inline_formatting.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/inline_internal_targets.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/line_block_indented.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/list_quote_paragraph.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/literal_block.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/literal_block_extra.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/literal_block_quoted.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/named_reference.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/non_aligned_text.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/option_list.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/option_list_complex.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/paragraph.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/simple_table.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/simple_tables_complex.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/substitutions_complex.tokens
==============================================================================
Binary files - no diff available.
--
svn-components mailing list
[EMAIL PROTECTED]
http://lists.ez.no/mailman/listinfo/svn-components
