Author: kn
Date: Thu Feb 14 14:07:17 2008
New Revision: 7373
Log:
- Added end of file token
Modified:
experimental/Document/src/document/rst/token.php
experimental/Document/src/document/rst/tokenizer.php
experimental/Document/tests/document_rst_tokenizer_tests.php
experimental/Document/tests/files/rst/tokenizer/additional_references.tokens
experimental/Document/tests/files/rst/tokenizer/blockquote.tokens
experimental/Document/tests/files/rst/tokenizer/blockquote_attribution.tokens
experimental/Document/tests/files/rst/tokenizer/blockquote_list.tokens
experimental/Document/tests/files/rst/tokenizer/blockquote_multiple.tokens
experimental/Document/tests/files/rst/tokenizer/blockquote_multiple_attribution.tokens
experimental/Document/tests/files/rst/tokenizer/bullet_list.tokens
experimental/Document/tests/files/rst/tokenizer/bullet_list_deep.tokens
experimental/Document/tests/files/rst/tokenizer/bullet_list_incorrect.tokens
experimental/Document/tests/files/rst/tokenizer/citation.tokens
experimental/Document/tests/files/rst/tokenizer/citation_full.tokens
experimental/Document/tests/files/rst/tokenizer/colons.tokens
experimental/Document/tests/files/rst/tokenizer/comment.tokens
experimental/Document/tests/files/rst/tokenizer/comments_complex.tokens
experimental/Document/tests/files/rst/tokenizer/deep_block_quote.tokens
experimental/Document/tests/files/rst/tokenizer/definition_list.tokens
experimental/Document/tests/files/rst/tokenizer/definition_list_alignements.tokens
experimental/Document/tests/files/rst/tokenizer/definition_list_classifier.tokens
experimental/Document/tests/files/rst/tokenizer/directive.tokens
experimental/Document/tests/files/rst/tokenizer/directives_complex.tokens
experimental/Document/tests/files/rst/tokenizer/embedded_uris.tokens
experimental/Document/tests/files/rst/tokenizer/empty.tokens
experimental/Document/tests/files/rst/tokenizer/enumerated_list.tokens
experimental/Document/tests/files/rst/tokenizer/enumerated_list_deep.tokens
experimental/Document/tests/files/rst/tokenizer/enumerated_list_not.tokens
experimental/Document/tests/files/rst/tokenizer/escaping.tokens
experimental/Document/tests/files/rst/tokenizer/field_list.tokens
experimental/Document/tests/files/rst/tokenizer/field_list_indented.tokens
experimental/Document/tests/files/rst/tokenizer/footnote.tokens
experimental/Document/tests/files/rst/tokenizer/footnote_complex.tokens
experimental/Document/tests/files/rst/tokenizer/footnotes.tokens
experimental/Document/tests/files/rst/tokenizer/footnotes_asterisk.tokens
experimental/Document/tests/files/rst/tokenizer/footnotes_auto.tokens
experimental/Document/tests/files/rst/tokenizer/grid_tables.tokens
experimental/Document/tests/files/rst/tokenizer/grid_tables_complex.tokens
experimental/Document/tests/files/rst/tokenizer/hyperlink_targets.tokens
experimental/Document/tests/files/rst/tokenizer/hyperlink_targets_complex.tokens
experimental/Document/tests/files/rst/tokenizer/hyperlinks.tokens
experimental/Document/tests/files/rst/tokenizer/inline_formatting.tokens
experimental/Document/tests/files/rst/tokenizer/inline_internal_targets.tokens
experimental/Document/tests/files/rst/tokenizer/line_block.tokens
experimental/Document/tests/files/rst/tokenizer/line_block_indented.tokens
experimental/Document/tests/files/rst/tokenizer/list_quote_paragraph.tokens
experimental/Document/tests/files/rst/tokenizer/literal_block.tokens
experimental/Document/tests/files/rst/tokenizer/literal_block_extra.tokens
experimental/Document/tests/files/rst/tokenizer/literal_block_notations.tokens
experimental/Document/tests/files/rst/tokenizer/literal_block_quoted.tokens
experimental/Document/tests/files/rst/tokenizer/named_reference.tokens
experimental/Document/tests/files/rst/tokenizer/non_aligned_text.tokens
experimental/Document/tests/files/rst/tokenizer/option_list.tokens
experimental/Document/tests/files/rst/tokenizer/option_list_complex.tokens
experimental/Document/tests/files/rst/tokenizer/paragraph.tokens
experimental/Document/tests/files/rst/tokenizer/simple_table.tokens
experimental/Document/tests/files/rst/tokenizer/simple_tables_complex.tokens
experimental/Document/tests/files/rst/tokenizer/substitution.tokens
experimental/Document/tests/files/rst/tokenizer/substitutions_complex.tokens
experimental/Document/tests/files/rst/tokenizer/tabulators.tokens
experimental/Document/tests/files/rst/tokenizer/titles.tokens
experimental/Document/tests/files/rst/tokenizer/transistion.tokens
Modified: experimental/Document/src/document/rst/token.php
==============================================================================
--- experimental/Document/src/document/rst/token.php [iso-8859-1] (original)
+++ experimental/Document/src/document/rst/token.php [iso-8859-1] Thu Feb 14
14:07:17 2008
@@ -18,7 +18,7 @@
*/
class ezcDocumentRstToken extends ezcBaseStruct
{
- // Whitespace tokens
+ // Token type constants
const WHITESPACE = 1;
const NEWLINE = 2;
@@ -27,6 +27,8 @@
const SPECIAL_CHARS = 4;
const TEXT_LINE = 5;
+
+ const EOF = 6;
/**
* Token type
Modified: experimental/Document/src/document/rst/tokenizer.php
==============================================================================
--- experimental/Document/src/document/rst/tokenizer.php [iso-8859-1] (original)
+++ experimental/Document/src/document/rst/tokenizer.php [iso-8859-1] Thu Feb
14 14:07:17 2008
@@ -185,6 +185,12 @@
);
}
+ // Finally append a end of file token
+ $tokens[] = new ezcDocumentRstToken(
+ ezcDocumentRstToken::EOF,
+ null, $line, $position
+ );
+
return $tokens;
}
}
Modified: experimental/Document/tests/document_rst_tokenizer_tests.php
==============================================================================
--- experimental/Document/tests/document_rst_tokenizer_tests.php [iso-8859-1]
(original)
+++ experimental/Document/tests/document_rst_tokenizer_tests.php [iso-8859-1]
Thu Feb 14 14:07:17 2008
@@ -60,7 +60,7 @@
$expected = include $to;
// Store test file, to have something to compare on failure
- $tempDir = $this->createTempDir( 'rst_tokenizer' ) . '/';
+ $tempDir = $this->createTempDir( 'rst_tokenizer_' ) . '/';
file_put_contents( $tempDir . basename( $to ), "<?php\n\nreturn " .
var_export( $tokens, true ) . ";\n\n" );
$this->assertEquals(
Modified:
experimental/Document/tests/files/rst/tokenizer/additional_references.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/blockquote.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/blockquote_attribution.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/blockquote_list.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/blockquote_multiple.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/blockquote_multiple_attribution.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/bullet_list.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/bullet_list_deep.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/bullet_list_incorrect.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/citation.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/citation_full.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/colons.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/comment.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/comments_complex.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/deep_block_quote.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/definition_list.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/definition_list_alignements.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/definition_list_classifier.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/directive.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/directives_complex.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/embedded_uris.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/empty.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/enumerated_list.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/enumerated_list_deep.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/enumerated_list_not.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/escaping.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/field_list.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/field_list_indented.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/footnote.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/footnote_complex.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/footnotes.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/footnotes_asterisk.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/footnotes_auto.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/grid_tables.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/grid_tables_complex.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/hyperlink_targets.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/hyperlink_targets_complex.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/hyperlinks.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/inline_formatting.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/inline_internal_targets.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/line_block.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/line_block_indented.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/list_quote_paragraph.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/literal_block.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/literal_block_extra.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/literal_block_notations.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/literal_block_quoted.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/named_reference.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/non_aligned_text.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/option_list.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/option_list_complex.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/paragraph.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/simple_table.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/simple_tables_complex.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/substitution.tokens
==============================================================================
Binary files - no diff available.
Modified:
experimental/Document/tests/files/rst/tokenizer/substitutions_complex.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/tabulators.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/titles.tokens
==============================================================================
Binary files - no diff available.
Modified: experimental/Document/tests/files/rst/tokenizer/transistion.tokens
==============================================================================
Binary files - no diff available.
--
svn-components mailing list
svn-components@lists.ez.no
http://lists.ez.no/mailman/listinfo/svn-components
