Author: kn
Date: Mon Feb 11 12:15:30 2008
New Revision: 7337
Log:
- Added tab expansion
Added:
experimental/Document/tests/files/rst/tokenizer/tabulators.tokens (with
props)
experimental/Document/tests/files/rst/tokenizer/tabulators.txt (with
props)
Modified:
experimental/Document/src/document/rst/tokenizer.php
experimental/Document/tests/files/rst/tokenizer/colons.tokens
Modified: experimental/Document/src/document/rst/tokenizer.php
==============================================================================
--- experimental/Document/src/document/rst/tokenizer.php [iso-8859-1] (original)
+++ experimental/Document/src/document/rst/tokenizer.php [iso-8859-1] Mon Feb
11 12:15:30 2008
@@ -98,6 +98,26 @@
}
/**
+ * Convert tabs to spaces
+ *
+ * Convert all tabs to spaces, as defined in:
+ *
http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#whitespace
+ *
+ * @param ezcDocumentRstToken $token
+ * @return void
+ */
+ protected function convertTabs( ezcDocumentRstToken $token )
+ {
+ while ( ( $position = strpos( $token->content, "\t" ) ) !== false )
+ {
+ $token->content =
+ substr( $token->content, 0, $position ) .
+ str_repeat( ' ', 9 - ( ( $position + $token->position ) % 8 )
) .
+ substr( $token->content, $position + 1 );
+ }
+ }
+
+ /**
* Tokenize the given string
*
* The method tries to tokenize the passed strings and returns an array of
@@ -122,7 +142,7 @@
{
// A token matched, so add the matched token to the token
// list and update all variables.
- $tokens[] = new ezcDocumentRstToken(
+ $newToken = new ezcDocumentRstToken(
$token,
( isset( $matches['value'] ) ? $matches['value'] :
null ),
$line,
@@ -139,6 +159,15 @@
++$line;
$position = 1;
}
+
+ // Convert tabs to spaces for whitespace tokens
+ if ( $newToken->type == ezcDocumentRstToken::WHITESPACE )
+ {
+ $this->convertTabs( $newToken );
+ }
+
+ // Add token to extracted token list
+ $tokens[] = $newToken;
// Restart the while loop, because we matched a token and
// can retry with shortened string.
Modified: experimental/Document/tests/files/rst/tokenizer/colons.tokens
==============================================================================
Binary files - no diff available.
Added: experimental/Document/tests/files/rst/tokenizer/tabulators.tokens
==============================================================================
Binary file - no diff available.
Propchange: experimental/Document/tests/files/rst/tokenizer/tabulators.tokens
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: experimental/Document/tests/files/rst/tokenizer/tabulators.txt
==============================================================================
--- experimental/Document/tests/files/rst/tokenizer/tabulators.txt (added)
+++ experimental/Document/tests/files/rst/tokenizer/tabulators.txt [iso-8859-1]
Mon Feb 11 12:15:30 2008
@@ -1,0 +1,12 @@
+ test
+ test
+ test
+ test
+ test
+ test
+ test
+ test
+ test
+ test
+a test
+ a a test
Propchange: experimental/Document/tests/files/rst/tokenizer/tabulators.txt
------------------------------------------------------------------------------
svn:eol-style = native
--
svn-components mailing list
[EMAIL PROTECTED]
http://lists.ez.no/mailman/listinfo/svn-components
