edit: $/Dev10/feature/vs_langs01_s/Merlin/External.LCA_RESTRICTED/Languages/IronRuby/mspec/rubyspec/library/yaml/load_spec.rb;C1496258 File: load_spec.rb =================================================================== --- $/Dev10/feature/vs_langs01_s/Merlin/External.LCA_RESTRICTED/Languages/IronRuby/mspec/rubyspec/library/yaml/load_spec.rb;C1496258 (server) 1/28/2010 2:28 PM +++ Shelved Change: $/Dev10/feature/vs_langs01_s/Merlin/External.LCA_RESTRICTED/Languages/IronRuby/mspec/rubyspec/library/yaml/load_spec.rb;Y2 @@ -66,6 +66,41 @@ YAML.load("--- \n*.rb").should == "*.rb" YAML.load("--- \n&.rb").should == "&.rb" end + + it "terminates plain scalars with a space or a line break" do + YAML.load("{hello: uv}").should == { "hello" => "uv" } + YAML.load("{hello\txy:\tuv}").should == { "hello\txy:\tuv" => nil } + YAML.load("{hello\rxy:\ruv}").should == { "hello\rxy:\ruv" => nil } + + YAML.load("[hello,\nx,y]").should == ["hello", "x,y"] + YAML.load("[hello, ,,a]").should == ["hello", ",,a"] + YAML.load("[hello, x,y]").should == ["hello", "x,y"] + + YAML.load("[hello,\tx,y]").should == ["hello,\tx,y"] + YAML.load("[hello,\rx,y]").should == ["hello,\rx,y"] + end + + it "normalizes \\r\\n and \\n line breaks and removes the first one" do + YAML.load("str\r\nx").should == "str x" + YAML.load("str\r\n\r\nx").should == "str\nx" + YAML.load("str\r\n\r\n\r\nx").should == "str\n\nx" + + YAML.load("str\nx").should == "str x" + YAML.load("str\n\nx").should == "str\nx" + YAML.load("str\n\n\nx").should == "str\n\nx" + end + + it "trims whitespace if not followed by a non-whitespace" do + YAML.load("str \t\t\t\tx").should == "str \t\t\t\tx" + YAML.load("str\t\t\t\tx").should == "str\t\t\t\tx" + YAML.load("str\r\r\r\r").should == "str\r\r\r\r" + YAML.load("str x").should == "str x" + YAML.load("str \t ").should == "str" + YAML.load("str \t\t\r").should == "str \t\t\r" + YAML.load("str\nx").should == "str x" + YAML.load("str\rx").should == "str\rx" + YAML.load("str\x85").should == "str\205" + end it "works with block sequence shortcuts" do block_seq = "- - - one\n - two\n - three" =================================================================== edit: $/Dev10/feature/vs_langs01_s/Merlin/External.LCA_RESTRICTED/Languages/IronRuby/yaml/IronRuby.Libraries.Yaml/Engine/Parser.cs;C1496258 File: Parser.cs =================================================================== --- $/Dev10/feature/vs_langs01_s/Merlin/External.LCA_RESTRICTED/Languages/IronRuby/yaml/IronRuby.Libraries.Yaml/Engine/Parser.cs;C1496258 (server) 1/27/2010 5:51 PM +++ Shelved Change: $/Dev10/feature/vs_langs01_s/Merlin/External.LCA_RESTRICTED/Languages/IronRuby/yaml/IronRuby.Libraries.Yaml/Engine/Parser.cs;Y2 @@ -20,6 +20,7 @@ using System.Collections.Generic; using System.Text.RegularExpressions; using System.Text; +using System.Globalization; namespace IronRuby.StandardLibrary.Yaml { public class Parser : IEnumerable { @@ -95,6 +96,13 @@ _parseStack.Push(Production.STREAM); } + private void ReportError(string/*!*/ message, params object[]/*!*/ args) { + throw new ParserException( + String.Format(CultureInfo.InvariantCulture, message, args) + + String.Format(CultureInfo.InvariantCulture, " (line {0}, column {1})", _scanner.Line + 1, _scanner.Column) + ); + } + public Encoding/*!*/ Encoding { get { return _scanner.Encoding; } } @@ -127,7 +135,9 @@ } private YamlEvent Produce() { - switch (_parseStack.Pop()) { + var prod = _parseStack.Pop(); + //Console.WriteLine(prod); + switch (prod) { case Production.STREAM: { _parseStack.Push(Production.STREAM_END); _parseStack.Push(Production.EXPLICIT_DOCUMENT); @@ -167,7 +177,7 @@ Dictionary tags; ProcessDirectives(out version, out tags); if (!(_scanner.PeekToken() is DocumentStartToken)) { - throw new ParserException("expected '', but found: " + tok); + ReportError("expected '', but found: {0}", tok); } _scanner.GetToken(); return new DocumentStartEvent(true, version, tags); @@ -252,7 +262,7 @@ } else if (tok is ScalarToken) { _parseStack.Push(Production.SCALAR); } else { - throw new ParserException("while scanning a flow node: expected the node content, but found: " + tok); + ReportError("while scanning a flow node: expected the node content, but found: {0}", tok); } return null; } @@ -362,10 +372,9 @@ return new SequenceStartEvent(_anchors.First.Value, _tags.First.Value != "!" ? _tags.First.Value : null, FlowStyle.Block); } case Production.BLOCK_SEQUENCE_END: { - Token tok = null; - if (!(_scanner.PeekToken() is BlockEndToken)) { - tok = _scanner.PeekToken(); - throw new ParserException("while scanning a block collection: expected , but found: " + tok); + Token tok = _scanner.PeekToken(); + if (!(tok is BlockEndToken)) { + ReportError("while scanning a block collection: expected , but found: {0}", tok); } _scanner.GetToken(); return SequenceEndEvent.Instance; @@ -375,10 +384,9 @@ return new MappingStartEvent(_anchors.First.Value, _tags.First.Value != "!" ? _tags.First.Value : null, FlowStyle.Block); } case Production.BLOCK_MAPPING_END: { - Token tok = null; - if (!(_scanner.PeekToken() is BlockEndToken)) { - tok = _scanner.PeekToken(); - throw new ParserException("while scanning a block mapping: expected , but found: " + tok); + Token tok = _scanner.PeekToken(); + if (!(tok is BlockEndToken)) { + ReportError("while scanning a block mapping: expected , but found: {0}", tok); } _scanner.GetToken(); return MappingEndEvent.Instance; @@ -555,7 +563,7 @@ } if (handle != null) { if (!_tagHandles.ContainsKey(handle)) { - throw new ParserException("while parsing a node: found undefined tag handle :" + handle); + ReportError("while parsing a node: found undefined tag handle: {0}", handle); } if ((ix = suffix.IndexOf("/")) != -1) { string before = suffix.Substring(0, ix); @@ -583,12 +591,12 @@ DirectiveToken tok = (DirectiveToken)_scanner.GetToken(); if (tok.Name == "Yaml") { if (_yamlVersion != null) { - throw new ParserException("found duplicate Yaml directive"); + ReportError("found duplicate Yaml directive"); } int major = int.Parse(tok.Value[0]); int minor = int.Parse(tok.Value[1]); if (major != 1) { - throw new ParserException("found incompatible Yaml document (version 1.* is required)"); + ReportError("found incompatible Yaml document (version 1.* is required)"); } _yamlVersion = new Version(major, minor); @@ -596,7 +604,7 @@ string handle = tok.Value[0]; string prefix = tok.Value[1]; if (_tagHandles.ContainsKey(handle)) { - throw new ParserException("duplicate tag handle: " + handle); + ReportError("duplicate tag handle: {0}", handle); } _tagHandles.Add(handle, prefix); } =================================================================== edit: $/Dev10/feature/vs_langs01_s/Merlin/External.LCA_RESTRICTED/Languages/IronRuby/yaml/IronRuby.Libraries.Yaml/Engine/Scanner.cs;C1496258 File: Scanner.cs =================================================================== --- $/Dev10/feature/vs_langs01_s/Merlin/External.LCA_RESTRICTED/Languages/IronRuby/yaml/IronRuby.Libraries.Yaml/Engine/Scanner.cs;C1496258 (server) 1/27/2010 5:22 PM +++ Shelved Change: $/Dev10/feature/vs_langs01_s/Merlin/External.LCA_RESTRICTED/Languages/IronRuby/yaml/IronRuby.Libraries.Yaml/Engine/Scanner.cs;Y2 @@ -29,15 +29,11 @@ private class SimpleKey { internal readonly int TokenNumber; internal readonly bool Required; - internal readonly int Index; - internal readonly int Line; internal readonly int Column; - internal SimpleKey(int tokenNumber, bool required, int index, int line, int column) { + internal SimpleKey(int tokenNumber, bool required, int column) { TokenNumber = tokenNumber; Required = required; - Index = index; - Line = line; Column = column; } } @@ -49,6 +45,7 @@ private bool _allowSimpleKey = true; private bool _eof = false; private int _column = 0; + private int _line = 0; private bool _docStart = false; private readonly LinkedList _tokens = new LinkedList(); private readonly Stack _indents = new Stack(); @@ -66,10 +63,41 @@ FetchStreamStart(); } + private void ReportError(string/*!*/ message, params object[]/*!*/ args) { + throw new ScannerException( + String.Format(CultureInfo.InvariantCulture, message, args) + + String.Format(CultureInfo.InvariantCulture, " (line {0}, column {1})", _line + 1, _column) + ); + } + + private void ReportUnexpectedCharacter(string/*!*/ context, char expected) { + ReportUnexpectedCharacter(context, "`" + expected + "'"); + } + + private void ReportUnexpectedCharacter(string/*!*/ context, string/*!*/ expected) { + ReportError("while scanning {0}: expected {1}", context, expected); + } + + private void ReportUnexpectedCharacter(string/*!*/ context, char expected, char unexpected) { + ReportUnexpectedCharacter(context, "`" + expected + "'", unexpected); + } + + private void ReportUnexpectedCharacter(string/*!*/ context, string/*!*/ expected, char unexpected) { + ReportError("while scanning {0}: expected {1}, but found `{2}' ({3})", context, expected, unexpected, (int)unexpected); + } + private int FlowLevel { get { return _collectionTokens.Count; } } + public int Line { + get { return _line; } + } + + public int Column { + get { return _column; } + } + public Encoding/*!*/ Encoding { get { return _encoding; } } @@ -147,28 +175,26 @@ return _buffer[_pointer + index]; } - private void Forward() { - Ensure(2, true); - char ch1 = _buffer[_pointer++]; - if (ch1 == '\n' || (ch1 == '\r' && _buffer[_pointer] != '\n')) { + private void Advance() { + char c = _buffer[_pointer++]; + if (c == '\n' || (c == '\r' && _buffer[_pointer] != '\n')) { _possibleSimpleKeys.Clear(); _column = 0; + _line++; } else { _column++; } } + private void Forward() { + Ensure(2, true); + Advance(); + } + private void Forward(int length) { Ensure(length + 1, true); for (int i = 0; i < length; i++) { - char ch = _buffer[_pointer]; - _pointer++; - if (ch == '\n' || (ch == '\r' && _buffer[_pointer] != '\n')) { - _possibleSimpleKeys.Clear(); - _column = 0; - } else { - _column++; - } + Advance(); } } @@ -184,6 +210,7 @@ _tokensTaken++; Token t = _tokens.First.Value; _tokens.RemoveFirst(); + // Console.WriteLine(t); return t; } return null; @@ -242,72 +269,82 @@ } private Token FetchMoreTokens() { - ScanToNextToken(); - UnwindIndent(_column); - char ch = Peek(); - bool colz = _column == 0; - switch (ch) { - case '\0': return FetchStreamEnd(); - case '\'': return FetchSingle(); - case '"': return FetchDouble(); + while (true) { + ScanToNextToken(); + UnwindIndent(_column); + char c = Peek(); + char d; + bool atLineStart = _column == 0; + switch (c) { + case '\0': return FetchStreamEnd(); + case '\'': return FetchSingle(); + case '"': return FetchDouble(); - case '?': - if (FlowLevel != 0 || NULL_BL_T_LINEBR(Peek(1))) { - return FetchKey(); - } - break; + case '?': + if (FlowLevel != 0 || NULL_BL_T_LINEBR(Peek(1))) { + return FetchKey(); + } + break; - case ':': - if (FlowLevel != 0 || NULL_BL_T_LINEBR(Peek(1))) { - // key: value not allowed in a sequence [...]: - if (FlowLevel == 0 || _collectionTokens.Peek() != FlowSequenceStartToken.Instance) { - return FetchValue(); + case ':': + if (FlowLevel != 0 || NULL_BL_T_LINEBR(Peek(1))) { + // key: value not allowed in a sequence [...]: + if (FlowLevel == 0 || _collectionTokens.Peek() != FlowSequenceStartToken.Instance) { + return FetchValue(); + } } - } - break; + break; - case '%': - if (colz) { - return FetchDirective(); - } - break; + case '%': + if (atLineStart) { + return FetchDirective(); + } + break; - case '-': - if ((colz || _docStart) && IsEnding()) { - return FetchDocumentStart(); - } else if (NULL_BL_T_LINEBR(Peek(1))) { - return FetchBlockEntry(); - } - break; + case '-': + if ((atLineStart || _docStart) && IsEnding()) { + return FetchDocumentStart(); + } else if (NULL_BL_T_LINEBR(Peek(1))) { + return FetchBlockEntry(); + } + break; - case '.': - if (colz && IsStart()) { - return FetchDocumentEnd(); - } - break; + case '.': + if (atLineStart && IsStart()) { + return FetchDocumentEnd(); + } + break; - case '[': return FetchFlowCollectionStart(FlowSequenceStartToken.Instance); - case '{': return FetchFlowCollectionStart(FlowMappingStartToken.Instance); - case ']': return FetchFlowCollectionEnd(FlowSequenceEndToken.Instance); - case '}': return FetchFlowCollectionEnd(FlowMappingEndToken.Instance); - case ',': return FetchFlowEntry(); - case '*': return FetchAlias(); - case '&': return FetchAnchor(); - case '!': return FetchTag(); - case '|': if (FlowLevel == 0) { return FetchLiteral(); } break; - case '>': if (FlowLevel == 0) { return FetchFolded(); } break; - } + case '[': return FetchFlowCollectionStart(FlowSequenceStartToken.Instance); + case '{': return FetchFlowCollectionStart(FlowMappingStartToken.Instance); + case ']': return FetchFlowCollectionEnd(FlowSequenceEndToken.Instance); + case '}': return FetchFlowCollectionEnd(FlowMappingEndToken.Instance); + case ',': + d = Peek(1); + if (d == ' ' || d == '\n') { + return FetchFlowEntry(); + } + break; - // TODO: This is Ruby specific and should be an extension... - // Ruby symbols (starting with ':'): - char c2 = _buffer[_pointer]; - if (NOT_USEFUL_CHAR(c2) || - (Ensure(1, false) && (c2 == '-' || c2 == '?' || c2 == ':') && - !NULL_BL_T_LINEBR(_buffer[_pointer + 1]))) { - return FetchPlain(); - } + case '*': return FetchAlias(); + case '&': return FetchAnchor(); + case '!': return FetchTag(); + case '|': if (FlowLevel == 0) { return FetchLiteral(); } break; + case '>': if (FlowLevel == 0) { return FetchFolded(); } break; + } - throw new ScannerException("while scanning for the next token: found character " + ch + " (" + (int)ch + ") that cannot start any token"); + // TODO: This is Ruby specific and should be an extension... + // Ruby symbols (starting with ':'): + d = Peek(); + if (NOT_USEFUL_CHAR(d) || + (Ensure(1, false) && (d == '-' || d == '?' || d == ':') && + !NULL_BL_T_LINEBR(_buffer[_pointer + 1]))) { + return FetchPlain(); + } + + ReportError("while scanning for the next token: found character `{0}' ({1}) that cannot start any token", c, (int)c); + Forward(); + } } private Token FetchStreamStart() { @@ -345,22 +382,14 @@ } } - private string ScanLineBreak() { - // Transforms: - // '\r\n' : '\n' - // '\r' : '\n' - // '\n' : '\n' - // '\x85' : '\n' - // default : '' - char val = Peek(); - if (FULL_LINEBR(val)) { - Ensure(2, false); - if (_buffer[_pointer] == '\r' && _buffer[_pointer + 1] == '\n') { - Forward(2); - } else { - Forward(); - } + private string/*!*/ ScanLineBreak() { + char c = Peek(); + if (c == '\n') { + Forward(); return "\n"; + } else if (c == '\r' && Peek(1) == '\n') { + Forward(2); + return "\n"; } else { return ""; } @@ -394,7 +423,7 @@ _docStart = false; if (FlowLevel == 0) { if (!_allowSimpleKey) { - throw new ScannerException("sequence entries are not allowed here"); + ReportError("sequence entries are not allowed here"); } if (AddIndent(_column)) { _tokens.AddLast(BlockSequenceStartToken.Instance); @@ -427,7 +456,7 @@ if (_possibleSimpleKeys.TryGetValue(FlowLevel, out key)) { _possibleSimpleKeys.Remove(FlowLevel); if (key.Required) { - throw new ScannerException("while scanning a simple key: could not find expected ':'"); + ReportUnexpectedCharacter("simple key", ':'); } } } @@ -435,7 +464,7 @@ private void SavePossibleSimpleKey() { if (_allowSimpleKey) { RemovePossibleSimpleKey(); - _possibleSimpleKeys.Add(FlowLevel, new SimpleKey(_tokensTaken + _tokens.Count, (FlowLevel == 0) && _indent == _column, -1, -1, _column)); + _possibleSimpleKeys.Add(FlowLevel, new SimpleKey(_tokensTaken + _tokens.Count, (FlowLevel == 0) && _indent == _column, _column)); } } @@ -447,7 +476,7 @@ Forward(2); suffix = ScanTagUri("tag"); if (Peek() != '>') { - throw new ScannerException("while scanning a tag: expected '>', but found " + Peek() + "(" + (int)Peek() + ")"); + ReportError("tag", '>', Peek()); } Forward(); } else if (NULL_BL_T_LINEBR(ch)) { @@ -474,7 +503,7 @@ suffix = ScanTagUri("tag"); } if (!NULL_BL_LINEBR(Peek())) { - throw new ScannerException("while scanning a tag: expected ' ', but found " + Peek() + "(" + (int)Peek() + ")"); + ReportUnexpectedCharacter("tag", ' ', Peek()); } return new TagToken(handle, suffix); } @@ -501,7 +530,7 @@ Forward(length); } if (chunks.Length == 0) { - throw new ScannerException("while scanning a " + name + ": expected URI, but found " + ch + "(" + (int)ch + ")"); + ReportUnexpectedCharacter(name, "URI", ch); } return chunks.ToString(); } @@ -509,7 +538,7 @@ private string ScanTagHandle(string name) { char ch = Peek(); if (ch != '!') { - throw new ScannerException("while scanning a " + name + ": expected '!', but found " + ch + "(" + (int)ch + ")"); + ReportUnexpectedCharacter(name, "!", ch); } int length = 1; ch = Peek(length); @@ -520,7 +549,7 @@ } if ('!' != ch) { Forward(length); - throw new ScannerException("while scanning a " + name + ": expected '!', but found " + ch + "(" + ((int)ch) + ")"); + ReportUnexpectedCharacter(name, '!', ch); } length++; } @@ -537,7 +566,10 @@ Ensure(2, false); str.Append(int.Parse(new string(_buffer, _pointer, 2), NumberStyles.HexNumber)); } catch (FormatException fe) { - throw new ScannerException("while scanning a " + name + ": expected URI escape sequence of 2 hexadecimal numbers, but found " + Peek(1) + "(" + ((int)Peek(1)) + ") and " + Peek(2) + "(" + ((int)Peek(2)) + ")", fe); + ReportError( + "while scanning a {0}: expected URI escape sequence of 2 hexadecimal numbers, but found `{0}' ({1}) and `{2}' ({3})", + name, Peek(1), Peek(2) + ); } Forward(2); } @@ -556,38 +588,14 @@ StringBuilder chunks = new StringBuilder(7); string spaces = ""; int ind = _indent + 1; - bool f_nzero; - CharTest r_check, r_check2, r_check3; - if (FlowLevel != 0) { - f_nzero = true; - r_check = R_FLOWNONZERO; - r_check2 = ALL_FALSE; - r_check3 = ALL_FALSE; - } else { - f_nzero = false; - r_check = NULL_BL_T_LINEBR; - r_check2 = R_FLOWZERO1; - r_check3 = NULL_BL_T_LINEBR; - } + while (Peek() != '#') { - int length = 0; - for (int i = 0; ; i++) { - Ensure(i + 2, false); - if (r_check(_buffer[_pointer + i]) || (r_check2(_buffer[_pointer + i]) && r_check3(_buffer[_pointer + i + 1]))) { - length = i; - char ch = Peek(length); - if (!(f_nzero && ch == ':' && !S4(Peek(length + 1)))) { - break; - } - } - } - + int length = FindEndOfPlain(); if (length == 0) { break; } _allowSimpleKey = false; chunks.Append(spaces); - Ensure(length, false); chunks.Append(_buffer, _pointer, length); Forward(length); spaces = ScanPlainSpaces(ind); @@ -598,6 +606,78 @@ return new ScalarToken(chunks.ToString(), ScalarQuotingStyle.None); } + private int FindEndOfPlain() { + int i = 0; + while (true) { + switch (Peek(i)) { + case '\t': + case '\0': + case ' ': + case '\n': + return i; + + case '\r': + if (Peek(i + 1) == '\n') { + return i; + } + break; + + case '[': + case ']': + case '{': + case '}': + case '?': + if (FlowLevel != 0) { + return i; + } + break; + + case ',': + if (FlowLevel != 0) { + switch (Peek(i + 1)) { + case '\0': + case ' ': + case '\n': + return i; + + case '\r': + if (Peek(i + 2) == '\n') { + return i; + } + break; + } + } + break; + + case ':': + switch (Peek(i + 1)) { + case '\0': + case ' ': + case '\n': + return i; + + case '\r': + if (Peek(i + 2) == '\n') { + return i; + } + break; + + case '[': + case ']': + case '{': + case '}': + if (FlowLevel != 0) { + return i; + } + break; + } + break; + } + + i++; + } + } + private int NextPossibleSimpleKey() { foreach (SimpleKey key in _possibleSimpleKeys.Values) { if (key.TokenNumber > 0) { @@ -609,34 +689,45 @@ private string ScanPlainSpaces(int indent) { StringBuilder chunks = new StringBuilder(); + int length = 0; - while (Peek(length) == ' ') { + char c; + while ((c = Peek(length)) == ' ' || c == '\t') { length++; } + char[] whitespaces = new char[length]; for (int i = 0; i < length; i++) { - whitespaces[i] = ' '; + whitespaces[i] = _buffer[_pointer + i]; } Forward(length); - char ch = Peek(); - if (FULL_LINEBR(ch)) { - string lineBreak = ScanLineBreak(); + + string lineBreak = ScanLineBreak(); + if (lineBreak.Length > 0) { _allowSimpleKey = true; if (IsEndOrStart()) { return ""; } + StringBuilder breaks = new StringBuilder(); - while (BLANK_OR_LINEBR(Peek())) { - if (' ' == Peek()) { + while (true) { + c = Peek(); + if (c == ' ' || c == '\t') { Forward(); } else { - breaks.Append(ScanLineBreak()); + var br = ScanLineBreak(); + if (br.Length == 0) { + break; + } + + breaks.Append(br); if (IsEndOrStart()) { return ""; } } } - if (!(lineBreak.Length == 1 && lineBreak[0] == '\n')) { + + if (lineBreak.Length != 1 || lineBreak[0] != '\n') { chunks.Append(lineBreak); } else if (breaks.Length == 0) { chunks.Append(" "); @@ -685,8 +776,12 @@ char c; try { c = (char)uint.Parse(str, NumberStyles.HexNumber); - } catch (Exception e) { - throw new ScannerException("while scanning a double-quoted scalar: expected escape sequence of " + length + " hexadecimal numbers, but found something else: " + str, e); + } catch (Exception) { + c = ' '; + ReportError( + "while scanning a double-quoted scalar: expected escape sequence of {0} hexadecimal numbers, but found `{1}'", + length, str + ); } Forward(length); @@ -744,7 +839,7 @@ Forward(length); char ch = Peek(); if (ch == '\0') { - throw new ScannerException("while scanning a quoted scalar: found unexpected end of stream"); + ReportError("while scanning a quoted scalar: found unexpected end of stream"); } else if (FULL_LINEBR(ch)) { string lineBreak = ScanLineBreak(); string breaks = ScanFlowScalarBreaks(); @@ -764,7 +859,7 @@ bool colz = true; for (; ; ) { if (colz && IsEndOrStart()) { - throw new ScannerException("while scanning a quoted scalar: found unexpected document separator"); + ReportError("while scanning a quoted scalar: found unexpected document separator"); } while (BLANK_T(Peek())) { Forward(); @@ -787,7 +882,7 @@ SimpleKey key; if (!_possibleSimpleKeys.TryGetValue(FlowLevel, out key)) { if (FlowLevel == 0 && !_allowSimpleKey) { - throw new ScannerException("mapping values are not allowed here"); + ReportError("mapping values are not allowed here"); } _allowSimpleKey = FlowLevel == 0; RemovePossibleSimpleKey(); @@ -976,14 +1071,14 @@ if (char.IsDigit(ch)) { increment = ch - '0'; if (increment == 0) { - throw new ScannerException("while scanning a block scalar: expected indentation indicator in_ the range 1-9, but found 0"); + ReportError("while scanning a block scalar: expected indentation indicator in_ the range 1-9, but found 0"); } Forward(); } } else if (Char.IsDigit(ch)) { increment = ch - '0'; if (increment == 0) { - throw new ScannerException("while scanning a block scalar: expected indentation indicator in_ the range 1-9, but found 0"); + ReportError("while scanning a block scalar: expected indentation indicator in_ the range 1-9, but found 0"); } Forward(); ch = Peek(); @@ -993,7 +1088,7 @@ } } if (!NULL_BL_LINEBR(Peek())) { - throw new ScannerException("while scanning a block scalar: expected chomping or indentation indicators, but found " + Peek() + "(" + ((int)Peek()) + ")"); + ReportUnexpectedCharacter("block scalar", "chomping or indentation indicators", Peek()); } } @@ -1026,7 +1121,7 @@ private Token FetchKey() { if (FlowLevel == 0) { if (!_allowSimpleKey) { - throw new ScannerException("mapping keys are not allowed here"); + ReportError("mapping keys are not allowed here"); } if (AddIndent(_column)) { _tokens.AddLast(BlockMappingStartToken.Instance); @@ -1077,7 +1172,7 @@ ch = Peek(length); } if (zlen) { - throw new ScannerException("while scanning a directive: expected alphabetic or numeric character, but found " + ch + "(" + ((int)ch) + ")"); + ReportUnexpectedCharacter("directive", "alphabetic or numeric character", ch); } string value = null; try { @@ -1087,7 +1182,7 @@ } Forward(length); if (!NULL_BL_LINEBR(Peek())) { - throw new ScannerException("while scanning a directive: expected alphabetic or numeric character, but found " + ch + "(" + ((int)ch) + ")"); + ReportUnexpectedCharacter("directive", "expected alphabetic or numeric character", ch); } return value; } @@ -1103,7 +1198,7 @@ } char ch = Peek(); if (!NULL_OR_LINEBR(ch)) { - throw new ScannerException("while scanning a directive: expected a comment or a line break, but found " + Peek() + "(" + ((int)Peek()) + ")"); + ReportUnexpectedCharacter("directive", "a comment or a line break", Peek()); } return ScanLineBreak(); } @@ -1117,7 +1212,7 @@ length++; } if (length == 0) { - throw new ScannerException("while scanning an " + name + ": expected alphabetic or numeric character, but found something else..."); + ReportUnexpectedCharacter(name, "alphabetic or numeric character"); } string value = null; try { @@ -1127,7 +1222,7 @@ } Forward(length); if (!NON_ALPHA_OR_NUM(Peek())) { - throw new ScannerException("while scanning an " + name + ": expected alphabetic or numeric character, but found " + Peek() + "(" + ((int)Peek()) + ")"); + ReportUnexpectedCharacter(name, "alphabetic or numeric character", Peek()); } return value; } @@ -1138,12 +1233,12 @@ } string major = ScanYamlDirectiveNumber(); if (Peek() != '.') { - throw new ScannerException("while scanning a directive: expected a digit or '.', but found " + Peek() + "(" + ((int)Peek()) + ")"); + ReportUnexpectedCharacter("directive", "a digit or a dot", Peek()); } Forward(); string minor = ScanYamlDirectiveNumber(); if (!NULL_BL_LINEBR(Peek())) { - throw new ScannerException("while scanning a directive: expected a digit or ' ', but found " + Peek() + "(" + ((int)Peek()) + ")"); + ReportUnexpectedCharacter("directive", "a digit or a space", Peek()); } return new string[] { major, minor }; } @@ -1151,7 +1246,7 @@ private string ScanYamlDirectiveNumber() { char ch = Peek(); if (!char.IsDigit(ch)) { - throw new ScannerException("while scanning a directive: expected a digit, but found " + ch + "(" + ((int)ch) + ")"); + ReportUnexpectedCharacter("directive", "a digit", ch); } int length = 0; StringBuilder sb = new StringBuilder(); @@ -1178,7 +1273,7 @@ private string ScanTagDirectiveHandle() { string value = ScanTagHandle("directive"); if (Peek() != ' ') { - throw new ScannerException("while scanning a directive: expected ' ', but found " + Peek() + "(" + ((int)Peek()) + ")"); + ReportUnexpectedCharacter("directive", ' ', Peek()); } return value; } @@ -1186,7 +1281,7 @@ private string ScanTagDirectivePrefix() { string value = ScanTagUri("directive"); if (!NULL_BL_LINEBR(Peek())) { - throw new ScannerException("while scanning a directive: expected ' ', but found " + Peek() + "(" + ((int)Peek()) + ")"); + ReportUnexpectedCharacter("directive", ' ', Peek()); } return value; } @@ -1297,49 +1392,6 @@ return true; } - private bool S4(char c) { - switch (c) { - case '\0': - case ' ': - case '\t': - case '\r': - case '\n': - case '[': - case ']': - case '{': - case '}': - return true; - } - return false; - } - - private bool ALL_FALSE(char c) { - return false; - } - - private bool R_FLOWZERO1(char c) { - return c == ':'; - } - - private bool R_FLOWNONZERO(char c) { - switch (c) { - case '\0': - case ' ': - case '\t': - case '\r': - case '\n': - case '[': - case ']': - case '{': - case '}': - case ',': - case ':': - case '?': - return true; - } - return false; - } - private bool ALPHA(char c) { return char.IsLetterOrDigit(c) || c == '-' || c == '_'; } =================================================================== edit: $/Dev10/feature/vs_langs01_s/Merlin/External.LCA_RESTRICTED/Languages/IronRuby/yaml/IronRuby.Libraries.Yaml/Engine/Tokens.cs;C1496258 File: Tokens.cs =================================================================== --- $/Dev10/feature/vs_langs01_s/Merlin/External.LCA_RESTRICTED/Languages/IronRuby/yaml/IronRuby.Libraries.Yaml/Engine/Tokens.cs;C1496258 (server) 1/28/2010 11:57 AM +++ Shelved Change: $/Dev10/feature/vs_langs01_s/Merlin/External.LCA_RESTRICTED/Languages/IronRuby/yaml/IronRuby.Libraries.Yaml/Engine/Tokens.cs;Y2 @@ -17,6 +17,7 @@ ***** END LICENSE BLOCK *****/ using System; +using System.Globalization; namespace IronRuby.StandardLibrary.Yaml { @@ -52,7 +53,7 @@ public string Value { get { return _value; } } public override string ToString() { - return string.Format("#", Value); + return String.Format(CultureInfo.InvariantCulture, "#", Value); } } @@ -79,6 +80,11 @@ public string Value { get { return _value; } } public ScalarQuotingStyle Style { get { return _style; } } + + public override string ToString() { + return String.Format(CultureInfo.InvariantCulture, "#", + Value, Style); + } } public sealed class DirectiveToken : Token { ===================================================================