This is an automated email from the ASF dual-hosted git repository. lkishalmi pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/netbeans.git
The following commit(s) were added to refs/heads/master by this push: new 88c0c39f20 TOML uses ANTLR4 Lexer Support 88c0c39f20 is described below commit 88c0c39f203c0815a0547ecbfba4d8d420615a17 Author: Laszlo Kishalmi <laszlo.kisha...@gmail.com> AuthorDate: Sun Jan 8 16:33:29 2023 -0800 TOML uses ANTLR4 Lexer Support --- ide/languages.toml/nbproject/project.xml | 8 ++ .../languages/toml/LexerInputCharStream.java | 135 --------------------- .../netbeans/modules/languages/toml/TomlLexer.java | 83 ++----------- .../modules/languages/toml/TomlTokenId.java | 3 +- 4 files changed, 22 insertions(+), 207 deletions(-) diff --git a/ide/languages.toml/nbproject/project.xml b/ide/languages.toml/nbproject/project.xml index ea7fe01d03..309960c9b0 100644 --- a/ide/languages.toml/nbproject/project.xml +++ b/ide/languages.toml/nbproject/project.xml @@ -114,6 +114,14 @@ <specification-version>1.80</specification-version> </run-dependency> </dependency> + <dependency> + <code-name-base>org.netbeans.modules.lexer.antlr4</code-name-base> + <build-prerequisite/> + <compile-dependency/> + <run-dependency> + <specification-version>1.0</specification-version> + </run-dependency> + </dependency> <dependency> <code-name-base>org.netbeans.modules.parsing.api</code-name-base> <build-prerequisite/> diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java deleted file mode 100644 index 7f2c2877ed..0000000000 --- a/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.netbeans.modules.languages.toml; - -import org.antlr.v4.runtime.CharStream; -import static org.antlr.v4.runtime.IntStream.UNKNOWN_SOURCE_NAME; -import org.antlr.v4.runtime.misc.Interval; -import org.netbeans.spi.lexer.*; - -/** - * - * @author lkishalmi - */ -public class LexerInputCharStream implements CharStream { - private final LexerInput input; - - private int tokenMark = Integer.MAX_VALUE; - private int index = 0; - - public LexerInputCharStream(LexerInput input) { - this.input = input; - } - - @Override - public String getText(Interval intrvl) { - if (intrvl.a < tokenMark) { - throw new UnsupportedOperationException("Can't read before the last token end: " + tokenMark); - } - int start = intrvl.a - tokenMark; - int end = intrvl.b - tokenMark + 1; - int toread = end - start - input.readLength(); - for (int i = 0; i < toread; i++) { - input.read(); - } - String ret = String.valueOf(input.readText(start, end)); - if (toread > 0) { - input.backup(toread); - } - return ret; - } - - @Override - public void consume() { - read(); - } - - @Override - public int LA(int count) { - if (count == 0) { - return 0; //the behaviour is not defined - } - - int c = 0; - if (count > 0) { - for (int i = 0; i < count; i++) { - c = read(); - } - backup(count); - } else { - backup(count); - c = read(); - } - return c; - } - - //Marks are for buffering in ANTLR4, we do not really need them - @Override - public int mark() { - return -1; - } - - public void markToken() { - tokenMark = index; - } - - @Override - public void release(int marker) { - } - - @Override - public int index() { - return index; - } - - @Override - public void seek(int i) { - if (i < index()) { - backup(index() - i); - } else { - while (index() < i) { - if (read() == LexerInput.EOF) { - break; - } - } - } - } - - - private int read() { - int ret = input.read(); - index += 1; - return ret; - } - - private void backup(int count) { - index -= count; - input.backup(count); - } - - @Override - public int size() { - throw new UnsupportedOperationException("Stream size is unknown."); - } - - @Override - public String getSourceName() { - return UNKNOWN_SOURCE_NAME; - } -} diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java index 0311f79063..6b2414749c 100644 --- a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java +++ b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java @@ -20,58 +20,31 @@ package org.netbeans.modules.languages.toml; import org.antlr.v4.runtime.misc.IntegerStack; import org.netbeans.api.lexer.Token; -import org.netbeans.spi.lexer.Lexer; import org.netbeans.spi.lexer.LexerRestartInfo; -import org.netbeans.spi.lexer.TokenFactory; import static org.tomlj.internal.TomlLexer.*; import static org.netbeans.modules.languages.toml.TomlTokenId.*; +import org.netbeans.spi.lexer.antlr4.AbstractAntlrLexerBridge; /** * * @author lkishalmi */ -public final class TomlLexer implements Lexer<TomlTokenId> { - - private final TokenFactory<TomlTokenId> tokenFactory; - private final org.tomlj.internal.TomlLexer lexer; - private final LexerInputCharStream input; - - public TomlLexer(LexerRestartInfo<TomlTokenId> info) { - this.tokenFactory = info.tokenFactory(); - this.input = new LexerInputCharStream(info.input()); - try { - this.lexer = new org.tomlj.internal.TomlLexer(input); - if (info.state() != null) { - ((LexerState) info.state()).restore(lexer); - } - input.markToken(); - } catch (Throwable ex) { - ex.printStackTrace(); - throw ex; - } - } +public final class TomlLexer extends AbstractAntlrLexerBridge<org.tomlj.internal.TomlLexer, TomlTokenId> { - private org.antlr.v4.runtime.Token preFetchedToken = null; + public TomlLexer(LexerRestartInfo<TomlTokenId> info, org.tomlj.internal.TomlLexer lexer) { + super(info, lexer); + } @Override - public org.netbeans.api.lexer.Token<TomlTokenId> nextToken() { - org.antlr.v4.runtime.Token nextToken; - if (preFetchedToken != null) { - nextToken = preFetchedToken; - lexer.getInputStream().seek(preFetchedToken.getStopIndex() + 1); - preFetchedToken = null; - } else { - nextToken = lexer.nextToken(); - } - int tokenType = nextToken.getType(); - switch (tokenType) { + protected Token<TomlTokenId> mapToken(int antlrTokenType) { + switch (antlrTokenType) { case EOF: return null; case StringChar: - return collate(StringChar, STRING); - + return groupToken(STRING, StringChar); + case TripleQuotationMark: case TripleApostrophe: case QuotationMark: @@ -133,59 +106,27 @@ public final class TomlLexer implements Lexer<TomlTokenId> { } } - protected org.netbeans.api.lexer.Token<TomlTokenId> collate(int tokenType, TomlTokenId tokenId) { - preFetchedToken = lexer.nextToken(); - while (preFetchedToken.getType() == tokenType) { - preFetchedToken = lexer.nextToken(); - } - lexer.getInputStream().seek(preFetchedToken.getStartIndex()); - return token(tokenId); - } - @Override public Object state() { return new LexerState(lexer); } - @Override - public void release() { - } - - private Token<TomlTokenId> token(TomlTokenId id) { - input.markToken(); - return tokenFactory.createToken(id); - } - - private static class LexerState { - final int state; - final int mode; - final IntegerStack modes; - + private static class LexerState extends AbstractAntlrLexerBridge.LexerState<org.tomlj.internal.TomlLexer> { final int arrayDepth; final IntegerStack arrayDepthStack; LexerState(org.tomlj.internal.TomlLexer lexer) { - this.state= lexer.getState(); + super(lexer); - this.mode = lexer._mode; - this.modes = new IntegerStack(lexer._modeStack); this.arrayDepth = lexer.arrayDepth; this.arrayDepthStack = new IntegerStack(lexer.arrayDepthStack); } public void restore(org.tomlj.internal.TomlLexer lexer) { - lexer.setState(state); - lexer._modeStack.addAll(modes); - lexer._mode = mode; + super.restore(lexer); lexer.arrayDepth = arrayDepth; lexer.arrayDepthStack.addAll(arrayDepthStack); } - - @Override - public String toString() { - return String.valueOf(state); - } - } } diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlTokenId.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlTokenId.java index 8eef764616..e60e21194e 100644 --- a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlTokenId.java +++ b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlTokenId.java @@ -26,6 +26,7 @@ import org.netbeans.api.lexer.TokenId; import org.netbeans.spi.lexer.LanguageHierarchy; import org.netbeans.spi.lexer.Lexer; import org.netbeans.spi.lexer.LexerRestartInfo; +import org.netbeans.spi.lexer.antlr4.LexerInputCharStream; /** * @@ -66,7 +67,7 @@ public enum TomlTokenId implements TokenId { @Override protected Lexer<TomlTokenId> createLexer(LexerRestartInfo<TomlTokenId> info) { - return new TomlLexer(info); + return new TomlLexer(info, new org.tomlj.internal.TomlLexer(new LexerInputCharStream(info.input()))); } @Override --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@netbeans.apache.org For additional commands, e-mail: commits-h...@netbeans.apache.org For further information about the NetBeans mailing lists, visit: https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists