packages/stinst/parser: 2014-01-07 Paolo Bonzini <[email protected]> * RBParser.st: Move parsing of literal arrays to RBParser. The scanner now returns just a RBLiteralArrayStartToken. * RBToken.st: Add RBLiteralArrayStartToken. * RewriteTests.st: Do not inspect the RBLiteralNode that is returned in TestFormat>>#testArrayRewrite, this will be covered by the parser tests. * RBScannerTests.st: Add tests for ##(...). * RBParserTests.st: Add more literal array tests. --- packages/stinst/parser/ChangeLog | 11 +++ packages/stinst/parser/RBParser.st | 132 ++++++++++++++++++++----------- packages/stinst/parser/RBParserTests.st | 42 ++++++++-- packages/stinst/parser/RBScannerTests.st | 53 +++++++++++++ packages/stinst/parser/RBToken.st | 23 ++++++ packages/stinst/parser/RewriteTests.st | 2 - 6 files changed, 208 insertions(+), 55 deletions(-)
diff --git a/packages/stinst/parser/ChangeLog b/packages/stinst/parser/ChangeLog index a1c35cb..9f44c4d 100644 --- a/packages/stinst/parser/ChangeLog +++ b/packages/stinst/parser/ChangeLog @@ -1,5 +1,16 @@ 2014-01-07 Paolo Bonzini <[email protected]> + * RBParser.st: Move parsing of literal arrays to RBParser. The + scanner now returns just a RBLiteralArrayStartToken. + * RBToken.st: Add RBLiteralArrayStartToken. + * RewriteTests.st: Do not inspect the RBLiteralNode that is + returned in TestFormat>>#testArrayRewrite, this will be covered + by the parser tests. + * RBScannerTests.st: Add tests for ##(...). + * RBParserTests.st: Add more literal array tests. + +2014-01-07 Paolo Bonzini <[email protected]> + * RBFormatter.st: Add #acceptLiteralArrayNode:. * RBParseNodes.st: Implement RBLiteralArrayNode. diff --git a/packages/stinst/parser/RBParser.st b/packages/stinst/parser/RBParser.st index b953762..99b8d2b 100644 --- a/packages/stinst/parser/RBParser.st +++ b/packages/stinst/parser/RBParser.st @@ -538,9 +538,8 @@ Object subclass: RBParser [ parseNegatedNumber [ <category: 'private-parsing'> | token | - self step. token := currentToken. - (token value respondsTo: #negated) ifFalse: [ + token value isNumber ifFalse: [ ^self parserError: 'Number expected' ]. token value negative ifTrue: [ ^self parserError: 'Positive number expected' ]. @@ -562,17 +561,96 @@ Object subclass: RBParser [ currentToken isIdentifier ifTrue: [^self parsePrimitiveIdentifier]. currentToken isLiteral ifTrue: [^self parsePrimitiveLiteral]. (currentToken isBinary and: [ currentToken value == #- ]) - ifTrue: [^self parseNegatedNumber]. + ifTrue: [^self step; parseNegatedNumber]. currentToken isSpecial ifTrue: [currentToken value == $[ ifTrue: [^self parseBlock]. currentToken value == ${ ifTrue: [^self parseArrayConstructor]. currentToken value == $( ifTrue: [^self parseParenthesizedExpression]]. + currentToken isLiteralArrayStart ifTrue: [^self parseLiteralArray]. currentToken isPatternBlock ifTrue: [^self parsePatternBlock]. currentToken isOptimized ifTrue: [^self parseOptimizedExpression]. self parserError: 'Variable expected' ] + parseLiteralArray [ + <category: 'private-parsing'> + | arrayStream start stop | + arrayStream := WriteStream on: (Array new: 10). + start := currentToken start. + self step. + + [currentToken isSpecial and: [currentToken value == $)]] + whileFalse: + [arrayStream nextPut: self parseLiteralArrayParts]. + stop := currentToken stop. + self step. + ^RBLiteralArrayNode + left: start + nodes: arrayStream contents + right: stop + ] + + parseByteArray [ + "FIXME: it's ugly that this is both here for #( [1 2 3] ) + and in RBScanner for #[1 2 3]." + <category: 'private-parsing'> + | byteStream number start stop | + byteStream := WriteStream on: (ByteArray new: 100). + start := currentToken start. + + [self step. + currentToken isLiteral] whileTrue: + [number := currentToken value. + (number isInteger and: [number between: 0 and: 255]) + ifFalse: [self scannerError: 'Expecting 8-bit integer']. + byteStream nextPut: number]. + (currentToken isSpecial and: [ currentToken value == $] ]) + ifFalse: [self scannerError: ''']'' expected']. + stop := currentToken stop. + self step. "]" + ^RBLiteralNode literalToken: (RBLiteralToken + value: byteStream contents + start: start + stop: stop) + ] + + parseLiteralArrayParts [ + <category: 'private-parsing'> + currentToken isLiteral ifTrue: [^self parsePrimitiveLiteral]. + currentToken isIdentifier + ifTrue: + [| token value | + token := currentToken. + value := self parsePrimitiveIdentifier. + value isVariable ifTrue: [ + value := RBLiteralNode literalToken: (RBLiteralToken + value: token value asSymbol + start: token start + stop: token stop)]. + ^value]. + currentToken isBinary + ifTrue: [ + | token | + token := currentToken. + self step. + (token value == #- and: [token stop + 1 = currentToken start and: [ + currentToken isLiteral and: [ + currentToken value isNumber and: [ + currentToken value positive ]]]]) ifTrue: [^self parseNegatedNumber]. + ^RBLiteralNode literalToken: (RBLiteralToken + value: token value + start: token start + stop: token stop)]. + currentToken isSpecial + ifTrue: + [currentToken value == $[ ifTrue: [^self parseByteArray]. + currentToken value == $( ifTrue: [^self parseLiteralArray]]. + currentToken isLiteralArrayStart ifTrue: [^self parseLiteralArray]. + currentToken isOptimized ifTrue: [^self parseOptimizedExpression]. + ^self parserError: 'literal array element expected' + ] + parseResourceTag [ <category: 'private-parsing'> | start | @@ -1206,57 +1284,15 @@ Stream subclass: RBScanner [ characterType == #binary ifTrue: [^(self scanBinary: RBLiteralToken) stop: self previousStepPosition]. currentCharacter == $' ifTrue: [^self scanStringSymbol]. - currentCharacter == $( ifTrue: [^self scanLiteralArray]. + currentCharacter == $( ifTrue: [ + self step. + ^RBLiteralArrayStartToken start: tokenStart]. currentCharacter == $[ ifTrue: [^self scanByteArray]. currentCharacter == ${ ifTrue: [^self scanQualifier]. currentCharacter == $# ifTrue: [^self scanExtendedLiterals]. self scannerError: 'Expecting a literal type' ] - scanLiteralArray [ - <category: 'private-scanning'> - | arrayStream start | - arrayStream := WriteStream on: (Array new: 10). - self step. - start := tokenStart. - - [self stripSeparators. - tokenStart := stream position. - currentCharacter == $)] - whileFalse: - [arrayStream nextPut: self scanLiteralArrayParts. - buffer reset]. - self step. - ^RBLiteralToken - value: arrayStream contents - start: start - stop: self previousStepPosition - ] - - scanLiteralArrayParts [ - <category: 'private-scanning'> - currentCharacter == $# ifTrue: [^self scanLiteral]. - characterType == #alphabetic - ifTrue: - [| token value | - token := self scanSymbol. - value := token value. - value == #nil ifTrue: [token value: nil]. - value == #true ifTrue: [token value: true]. - value == #false ifTrue: [token value: false]. - ^token]. - (characterType == #digit - or: [currentCharacter == $- and: [(self classify: stream peek) == #digit]]) - ifTrue: [^self scanNumber]. - characterType == #binary - ifTrue: [^(self scanBinary: RBLiteralToken) stop: self previousStepPosition]. - currentCharacter == $' ifTrue: [^self scanLiteralString]. - currentCharacter == $$ ifTrue: [^self scanLiteralCharacter]. - currentCharacter == $( ifTrue: [^self scanLiteralArray]. - currentCharacter == $[ ifTrue: [^self scanByteArray]. - ^self scannerError: 'Unknown character in literal array' - ] - scanLiteralCharacter [ <category: 'private-scanning'> | token value char tokenStop | diff --git a/packages/stinst/parser/RBParserTests.st b/packages/stinst/parser/RBParserTests.st index f569821..f0df0d1 100644 --- a/packages/stinst/parser/RBParserTests.st +++ b/packages/stinst/parser/RBParserTests.st @@ -58,14 +58,46 @@ TestCase subclass: TestParser [ ] testLiteralArrayParsing [ - | node | + | node nested | node := RBParser parseExpression: '#(-3 -2 -16r1)'. - self assert: node value first = -3. - self assert: node value second = -2. - self assert: node value third = -1. + self assert: node nodes first value = -3. + self assert: node nodes second value = -2. + self assert: node nodes third value = -1. node := RBParser parseExpression: '#(16r-1)'. - self assert: node value first = -1. + self assert: node nodes first value = -1. + + node := RBParser parseExpression: '#(- 2)'. + self assert: node nodes first value = #-. + self assert: node nodes second value = 2. + + node := RBParser parseExpression: '#(true false nil foo)'. + self assert: node nodes first value = true. + self assert: node nodes second value = false. + self assert: node nodes third value = nil. + self assert: node nodes fourth value = #foo. + + node := RBParser parseExpression: '#(1 #[2 3] 4)'. + self assert: node nodes first value = 1. + self assert: node nodes second value = #[2 3]. + self assert: node nodes third value = 4. + + node := RBParser parseExpression: '#(1 [2 3] 4)'. + self assert: node nodes first value = 1. + self assert: node nodes second value = #[2 3]. + self assert: node nodes third value = 4. + + node := RBParser parseExpression: '#(1 (2 3) 4)'. + self assert: node nodes first value = 1. + nested := node nodes second. + self assert: nested nodes first value = 2. + self assert: nested nodes second value = 3. + self assert: node nodes third value = 4. + + node := RBParser parseExpression: '#(1 ##(2/3) 4)'. + self assert: node nodes first value = 1. + self assert: node nodes second class == RBOptimizedNode. + self assert: node nodes third value = 4. ] ] diff --git a/packages/stinst/parser/RBScannerTests.st b/packages/stinst/parser/RBScannerTests.st index 41ecfd9..451e876 100644 --- a/packages/stinst/parser/RBScannerTests.st +++ b/packages/stinst/parser/RBScannerTests.st @@ -46,5 +46,58 @@ TestCase subclass: TestScanner [ num := scanner next. self assert: num value = 3. ] + + testEmbeddedCompiletimeConstant [ + | scanner token value | + scanner := RBScanner on: '#(##(1))' readStream. + + "Token is the literal" + token := scanner next. + self assert: token isLiteralArrayStart. + + "Token is for optimized code" + token := scanner next. + self assert: token isOptimized. + + "Token is the '1'" + token := scanner next. + self assert: token isLiteral. + self assert: token value equals: 1. + + "Token is the ')'" + token := scanner next. + self assert: token isSpecial. + self assert: token value equals: $). + + "Token is the ')'" + token := scanner next. + self assert: token isSpecial. + self assert: token value equals: $). + + "And we are at the end" + self assert: scanner atEnd. + ] + + testDirectCompiletimeConstant [ + | scanner token | + scanner := RBScanner on: '##(1)' readStream. + + "Token is for optimized code" + token := scanner next. + self assert: token isOptimized. + + "Token is the '1'" + token := scanner next. + self assert: token isLiteral. + self assert: token value equals: 1. + + "Token is the ')'" + token := scanner next. + self assert: token isSpecial. + self assert: token value equals: $). + + "And we are at the end" + self assert: scanner atEnd. + ] ] diff --git a/packages/stinst/parser/RBToken.st b/packages/stinst/parser/RBToken.st index 399a6db..7c22c9c 100644 --- a/packages/stinst/parser/RBToken.st +++ b/packages/stinst/parser/RBToken.st @@ -95,6 +95,11 @@ Object subclass: RBToken [ ^false ] + isLiteralArrayStart [ + <category: 'testing'> + ^false + ] + isOptimized [ <category: 'testing'> ^false @@ -391,6 +396,24 @@ RBValueToken subclass: RBKeywordToken [ +RBToken subclass: RBLiteralArrayStartToken [ + + <category: 'Refactory-Scanner'> + <comment: nil> + + isLiteralArrayStart [ + <category: 'testing'> + ^true + ] + + length [ + <category: 'testing'> + ^2 + ] +] + + + RBToken subclass: RBOptimizedToken [ <category: 'Refactory-Scanner'> diff --git a/packages/stinst/parser/RewriteTests.st b/packages/stinst/parser/RewriteTests.st index 322ac38..05130a4 100644 --- a/packages/stinst/parser/RewriteTests.st +++ b/packages/stinst/parser/RewriteTests.st @@ -273,8 +273,6 @@ TestCase subclass: TestFormat [ testArrayRewrite [ | inp res | inp := RBParser parseExpression: '#(16r01 2r01 16rFF )'. - self assert: inp value = (Array with: 1 with: 1 with: 255). - res := RBFormatter new formatAll: (Array with: inp). self assert: res = '#(16r01 2r01 16rFF)'. ] -- 1.8.4.2 _______________________________________________ help-smalltalk mailing list [email protected] https://lists.gnu.org/mailman/listinfo/help-smalltalk
