This is an automated email from the ASF dual-hosted git repository. lkishalmi pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/netbeans.git
The following commit(s) were added to refs/heads/master by this push: new 89b7a0930d Initial implementation of ANTLR4 Lexer Support (#5206) 89b7a0930d is described below commit 89b7a0930dcf453bba5a0512baaf47de9fd42ed8 Author: Laszlo Kishalmi <laszlo.kisha...@gmail.com> AuthorDate: Sun Jan 8 15:56:13 2023 -0800 Initial implementation of ANTLR4 Lexer Support (#5206) * Initial implementation of ANTLR4 Lexer Support * More documentation on AbstractAntlrLexerBridge * Added lexer.antlr4 test to GH Workflows * Removed System.out.println from tests --- .github/workflows/main.yml | 3 + ide/lexer.antlr4/arch.xml | 1063 ++++++++++++++++++++ ide/lexer.antlr4/build.xml | 41 + ide/lexer.antlr4/manifest.mf | 3 + ide/lexer.antlr4/nbproject/project.properties | 22 + ide/lexer.antlr4/nbproject/project.xml | 61 ++ .../spi/lexer/antlr4/AbstractAntlrLexerBridge.java | 183 ++++ .../spi/lexer/antlr4/AntlrTokenSequence.java | 249 +++++ .../netbeans/spi/lexer/antlr4/Bundle.properties | 23 + .../spi/lexer/antlr4/LexerInputCharStream.java | 136 +++ .../src/org/antlr/grammars/dummy/DummyLexer.g4 | 67 ++ .../src/org/antlr/grammars/dummy/DummyLexer.java | 175 ++++ .../spi/lexer/antlr4}/AntlrTokenSequenceTest.java | 26 +- .../languages/antlr/AntlrTokenSequenceTest.java | 9 - nbbuild/cluster.properties | 1 + 15 files changed, 2040 insertions(+), 22 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6c94c85813..431442ba04 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -523,6 +523,9 @@ jobs: - name: ide/lexer run: ant $OPTS -f ide/lexer test + - name: ide/lexer.antlr4 + run: ant $OPTS -f ide/lexer.antlr4 test + - name: ide/lib.terminalemulator run: ant $OPTS -f ide/lib.terminalemulator test diff --git a/ide/lexer.antlr4/arch.xml b/ide/lexer.antlr4/arch.xml new file mode 100644 index 0000000000..ea212512fb --- /dev/null +++ b/ide/lexer.antlr4/arch.xml @@ -0,0 +1,1063 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<!DOCTYPE api-answers PUBLIC "-//NetBeans//DTD Arch Answers//EN" "../../nbbuild/antsrc/org/netbeans/nbbuild/Arch.dtd" [ + <!ENTITY api-questions SYSTEM "../../nbbuild/antsrc/org/netbeans/nbbuild/Arch-api-questions.xml"> +]> + +<api-answers + question-version="1.29" + author="lkisha...@apache.org" +> + + &api-questions; + + +<!-- + <question id="arch-overall" when="init"> + Describe the overall architecture. + <hint> + What will be API for + <a href="http://openide.netbeans.org/tutorial/api-design.html#design.apiandspi"> + clients and what support API</a>? + What parts will be pluggable? + How will plug-ins be registered? Please use <code><api type="export"/></code> + to describe your general APIs. + If possible please provide + simple diagrams. + </hint> + </question> +--> + <answer id="arch-overall"> + <ul> + <li> + <api group="ide" name="ANTLRLexerBridge" type="export" category="devel"> + <p> + This module provides a bridge between ANTLRv4 Lexers and NetBeans Lexer + infrastructure. + </p> + </api> + </li> + </ul> + </answer> + + + + + + +<!-- + <question id="arch-usecases" when="init"> + Describe the main <a href="http://openide.netbeans.org/tutorial/api-design.html#usecase"> + use cases</a> of the new API. Who will use it at + what circumstances and what will be the typical code to write + to use the module. + </question> +--> + <answer id="arch-usecases"> + <p> + New language support with ANTLRv4 Grammar. + </p> + </answer> + + + +<!-- + <question id="resources-read" when="final"> + Does your module read any resources from layers? For what purpose? + + <hint> + As this is some kind of intermodule dependency, it is a kind of API. + Please describe it and classify according to + <a href="http://openide.netbeans.org/tutorial/api-design.html#categories"> + common stability categories</a>. + </hint> + </question> +--> + <answer id="resources-read"> + <p> + None. + </p> + </answer> + + + <answer id="exec-property"> + <p> + None. + </p> + </answer> + + <answer id="resources-file"> + <p> + None. + </p> + </answer> + + + + +<!-- + <question id="arch-quality" when="init"> + How will the <a href="http://www.netbeans.org/community/guidelines/q-evangelism.html" shape="rect">quality</a> + of your code be tested and + how are future regressions going to be prevented? + <hint> + What kind of testing do + you want to use? How much functionality, in which areas, + should be covered by the tests? How you find out that your + project was successful? + </hint> + </question> +--> + <answer id="arch-quality"> + <p> + XXX no answer for arch-quality + </p> + </answer> + + + +<!-- + <question id="arch-time" when="init"> + What are the time estimates of the work? + <hint> + Please express your estimates of how long the design, implementation, + stabilization are likely to last. How many people will be needed to + implement this and what is the expected milestone by which the work should be + ready? + </hint> + </question> +--> + <answer id="arch-time"> + <p> + XXX no answer for arch-time + </p> + </answer> + + + +<!-- + <question id="arch-what" when="init"> + What is this project good for? + <hint> + Please provide here a few lines describing the project, + what problem it should solve, provide links to documentation, + specifications, etc. + </hint> + </question> +--> + <answer id="arch-what"> + <p> + XXX no answer for arch-what + </p> + </answer> + + + +<!-- + <question id="arch-where" when="impl"> + Where one can find sources for your module? + <hint> + Please provide link to the Hg web client at + http://hg.netbeans.org/ + or just use tag defaultanswer generate='here' + </hint> + </question> +--> + <answer id="arch-where"> + <defaultanswer generate='here' /> + </answer> + + + +<!-- + <question id="compat-deprecation" when="init"> + How the introduction of your project influences functionality + provided by previous version of the product? + <hint> + If you are planning to deprecate/remove/change any existing APIs, + list them here accompanied with the reason explaining why you + are doing so. + </hint> + </question> +--> + <answer id="compat-deprecation"> + <p> + XXX no answer for compat-deprecation + </p> + </answer> + + + +<!-- + <question id="compat-i18n" when="impl"> + Is your module correctly internationalized? + <hint> + Correct internationalization means that it obeys instructions + at <a href="http://www.netbeans.org/download/dev/javadoc/org-openide-modules/org/openide/modules/doc-files/i18n-branding.html" shape="rect"> + NetBeans I18N pages</a>. + </hint> + </question> +--> + <answer id="compat-i18n"> + <p> + XXX no answer for compat-i18n + </p> + </answer> + + + +<!-- + <question id="compat-standards" when="init"> + Does the module implement or define any standards? Is the + implementation exact or does it deviate somehow? + </question> +--> + <answer id="compat-standards"> + <p> + XXX no answer for compat-standards + </p> + </answer> + + + +<!-- + <question id="compat-version" when="impl"> + Can your module coexist with earlier and future + versions of itself? Can you correctly read all old settings? Will future + versions be able to read your current settings? Can you read + or politely ignore settings stored by a future version? + + <hint> + Very helpful for reading settings is to store version number + there, so future versions can decide whether how to read/convert + the settings and older versions can ignore the new ones. + </hint> + </question> +--> + <answer id="compat-version"> + <p> + XXX no answer for compat-version + </p> + </answer> + + + +<!-- + <question id="dep-jre" when="final"> + Which version of JRE do you need (1.2, 1.3, 1.4, etc.)? + <hint> + It is expected that if your module runs on 1.x that it will run + on 1.x+1 if no, state that please. Also describe here cases where + you run different code on different versions of JRE and why. + </hint> + </question> +--> + <answer id="dep-jre"> + <p> + XXX no answer for dep-jre + </p> + </answer> + + + +<!-- + <question id="dep-jrejdk" when="final"> + Do you require the JDK or is the JRE enough? + </question> +--> + <answer id="dep-jrejdk"> + <p> + XXX no answer for dep-jrejdk + </p> + </answer> + + + +<!-- + <question id="dep-nb" when="init"> + What other NetBeans projects and modules does this one depend on? + <hint> + Depending on other NetBeans projects influnces the ability of + users of your work to customize their own branded version of + NetBeans by enabling and disabling some modules. Too + much dependencies restrict this kind of customization. If that + is your case, then you may want to split your functionality into + pieces of autoload, eager and regular modules which can be + enabled independently. Usually the answer to this question + is generated from your <code>project.xml</code> file, but + if it is not guessed correctly, you can suppress it by + specifying <defaultanswer generate="none"/> and + write here your own. Please describe such projects as imported APIs using + the <code><api name="identification" type="import or export" category="stable" url="where is the description" /></code>. + By doing this information gets listed in the summary page of your + javadoc. + </hint> + </question> +--> + <answer id="dep-nb"> + <defaultanswer generate='here' /> + </answer> + + + +<!-- + <question id="dep-non-nb" when="init"> + What other projects outside NetBeans does this one depend on? + + <hint> + Depending on 3rd party libraries is always problematic, + especially if they are not open source, as that complicates + the licensing scheme of NetBeans. Please enumerate your + external dependencies here, so it is correctly understood since + the begining what are the legal implications of your project. + Also please note that + some non-NetBeans projects are packaged as NetBeans modules + (see <a href="http://libs.netbeans.org/" shape="rect">libraries</a>) and + it is preferred to use this approach when more modules may + depend and share such third-party libraries. + </hint> + </question> +--> + <answer id="dep-non-nb"> + <p> + XXX no answer for dep-non-nb + </p> + </answer> + + + +<!-- + <question id="dep-platform" when="init"> + On which platforms does your module run? Does it run in the same + way on each? + <hint> + If you plan any dependency on OS or any usage of native code, + please describe why you are doing so and describe how you envision + to enforce the portability of your code. + Please note that there is a support for <a href="http://www.netbeans.org/download/dev/javadoc/org-openide-modules/org/openide/modules/doc-files/api.html#how-os-specific" shape="rect">OS conditionally + enabled modules</a> which together with autoload/eager modules + can allow you to enable to provide the best OS aware support + on certain OSes while providing compatibility bridge on the not + supported ones. + Also please list the supported + OSes/HW platforms and mentioned the lovest version of JDK required + for your project to run on. Also state whether JRE is enough or + you really need JDK. + </hint> + </question> +--> + <answer id="dep-platform"> + <p> + XXX no answer for dep-platform + </p> + </answer> + + + +<!-- + <question id="deploy-dependencies" when="final"> + What do other modules need to do to declare a dependency on this one, + in addition to or instead of the normal module dependency declaration + (e.g. tokens to require)? + <hint> + Provide a sample of the actual lines you would add to a module manifest + to declare a dependency, for example OpenIDE-Module-Requires: some.token. + If other modules should not depend on this module, or should just use a + simple regular module dependency, you can just answer "nothing". If you + intentionally expose a semistable API to clients using implementation + dependencies, you should mention that here (but there is no need to give + an example of usage). + </hint> + </question> +--> + <answer id="deploy-dependencies"> + <p> + XXX no answer for deploy-dependencies + </p> + </answer> + + + +<!-- + <question id="deploy-jar" when="impl"> + Do you deploy just module JAR file(s) or other files as well? + <hint> + Usually a module consist of one JAR file (perhaps with Class-Path + extensions) and also a configuration file that enables it. If you + have any other files, use + <api group="java.io.File" name="yourname" type="export" category="friend">...</api> + to define the location, name and stability of your files (of course + changing "yourname" and "friend" to suit your needs). + + If it uses more than one JAR, describe where they are located, how + they refer to each other. + If it consist of module JAR(s) and other files, please describe + what is their purpose, why other files are necessary. Please + make sure that installation/uninstallation leaves the system + in state as it was before installation. + </hint> + </question> +--> + <answer id="deploy-jar"> + <p> + XXX no answer for deploy-jar + </p> + </answer> + + + +<!-- + <question id="deploy-nbm" when="impl"> + Can you deploy an NBM via the Update Center? + <hint> + If not why? + </hint> + </question> +--> + <answer id="deploy-nbm"> + <p> + XXX no answer for deploy-nbm + </p> + </answer> + + + +<!-- + <question id="deploy-packages" when="init"> + Are packages of your module made inaccessible by not declaring them + public? + + <hint> + By default NetBeans build harness treats all packages are private. + If you export some of them - either as public or friend packages, + you should have a reason. If the reason is described elsewhere + in this document, you can ignore this question. + </hint> + </question> +--> + <answer id="deploy-packages"> + <p> + XXX no answer for deploy-packages + </p> + </answer> + + + +<!-- + <question id="deploy-shared" when="final"> + Do you need to be installed in the shared location only, or in the user directory only, + or can your module be installed anywhere? + <hint> + Installation location shall not matter, if it does explain why. + Consider also whether <code>InstalledFileLocator</code> can help. + </hint> + </question> +--> + <answer id="deploy-shared"> + <p> + XXX no answer for deploy-shared + </p> + </answer> + + + +<!-- + <question id="exec-ant-tasks" when="impl"> + Do you define or register any ant tasks that other can use? + + <hint> + If you provide an ant task that users can use, you need to be very + careful about its syntax and behaviour, as it most likely forms an + API for end users and as there is a lot of end users, their reaction + when such API gets broken can be pretty strong. + </hint> + </question> +--> + <answer id="exec-ant-tasks"> + <p> + XXX no answer for exec-ant-tasks + </p> + </answer> + + + +<!-- + <question id="exec-classloader" when="impl"> + Does your code create its own class loader(s)? + <hint> + A bit unusual. Please explain why and what for. + </hint> + </question> +--> + <answer id="exec-classloader"> + <p> + XXX no answer for exec-classloader + </p> + </answer> + + + +<!-- + <question id="exec-component" when="impl"> + Is execution of your code influenced by any (string) property + of any of your components? + + <hint> + Often <code>JComponent.getClientProperty</code>, <code>Action.getValue</code> + or <code>PropertyDescriptor.getValue</code>, etc. are used to influence + a behavior of some code. This of course forms an interface that should + be documented. Also if one depends on some interface that an object + implements (<code>component instanceof Runnable</code>) that forms an + API as well. + </hint> + </question> +--> + <answer id="exec-component"> + <p> + XXX no answer for exec-component + </p> + </answer> + + + +<!-- + <question id="exec-introspection" when="impl"> + Does your module use any kind of runtime type information (<code>instanceof</code>, + work with <code>java.lang.Class</code>, etc.)? + <hint> + Check for cases when you have an object of type A and you also + expect it to (possibly) be of type B and do some special action. That + should be documented. The same applies on operations in meta-level + (Class.isInstance(...), Class.isAssignableFrom(...), etc.). + </hint> + </question> +--> + <answer id="exec-introspection"> + <p> + XXX no answer for exec-introspection + </p> + </answer> + + + +<!-- + <question id="exec-privateaccess" when="final"> + Are you aware of any other parts of the system calling some of + your methods by reflection? + <hint> + If so, describe the "contract" as an API. Likely private or friend one, but + still API and consider rewrite of it. + </hint> + </question> +--> + <answer id="exec-privateaccess"> + <p> + XXX no answer for exec-privateaccess + </p> + </answer> + + + +<!-- + <question id="exec-process" when="impl"> + Do you execute an external process from your module? How do you ensure + that the result is the same on different platforms? Do you parse output? + Do you depend on result code? + <hint> + If you feed an input, parse the output please declare that as an API. + </hint> + </question> +--> + <answer id="exec-process"> + <p> + XXX no answer for exec-process + </p> + </answer> + + + +<!-- + <question id="exec-reflection" when="impl"> + Does your code use Java Reflection to execute other code? + <hint> + This usually indicates a missing or insufficient API in the other + part of the system. If the other side is not aware of your dependency + this contract can be easily broken. + </hint> + </question> +--> + <answer id="exec-reflection"> + <p> + XXX no answer for exec-reflection + </p> + </answer> + + + +<!-- + <question id="exec-threading" when="init"> + What threading models, if any, does your module adhere to? How the + project behaves with respect to threading? + <hint> + Is your API threadsafe? Can it be accessed from any threads or + just from some dedicated ones? Any special relation to AWT and + its Event Dispatch thread? Also + if your module calls foreign APIs which have a specific threading model, + indicate how you comply with the requirements for multithreaded access + (synchronization, mutexes, etc.) applicable to those APIs. + If your module defines any APIs, or has complex internal structures + that might be used from multiple threads, declare how you protect + data against concurrent access, race conditions, deadlocks, etc., + and whether such rules are enforced by runtime warnings, errors, assertions, etc. + Examples: a class might be non-thread-safe (like Java Collections); might + be fully thread-safe (internal locking); might require access through a mutex + (and may or may not automatically acquire that mutex on behalf of a client method); + might be able to run only in the event queue; etc. + Also describe when any events are fired: synchronously, asynchronously, etc. + Ideas: <a href="http://core.netbeans.org/proposals/threading/index.html#recommendations" shape="rect">Threading Recommendations</a> (in progress) + </hint> + </question> +--> + <answer id="exec-threading"> + <p> + XXX no answer for exec-threading + </p> + </answer> + + + +<!-- + <question id="format-clipboard" when="impl"> + Which data flavors (if any) does your code read from or insert to + the clipboard (by access to clipboard on means calling methods on <code>java.awt.datatransfer.Transferable</code>? + + <hint> + Often Node's deal with clipboard by usage of <code>Node.clipboardCopy, Node.clipboardCut and Node.pasteTypes</code>. + Check your code for overriding these methods. + </hint> + </question> +--> + <answer id="format-clipboard"> + <p> + XXX no answer for format-clipboard + </p> + </answer> + + + +<!-- + <question id="format-dnd" when="impl"> + Which protocols (if any) does your code understand during Drag & Drop? + <hint> + Often Node's deal with clipboard by usage of <code>Node.drag, Node.getDropType</code>. + Check your code for overriding these methods. Btw. if they are not overridden, they + by default delegate to <code>Node.clipboardCopy, Node.clipboardCut and Node.pasteTypes</code>. + </hint> + </question> +--> + <answer id="format-dnd"> + <p> + XXX no answer for format-dnd + </p> + </answer> + + + +<!-- + <question id="format-types" when="impl"> + Which protocols and file formats (if any) does your module read or write on disk, + or transmit or receive over the network? Do you generate an ant build script? + Can it be edited and modified? + + <hint> + <p> + Files can be read and written by other programs, modules and users. If they influence + your behaviour, make sure you either document the format or claim that it is a private + api (using the <api> tag). + </p> + + <p> + If you generate an ant build file, this is very likely going to be seen by end users and + they will be attempted to edit it. You should be ready for that and provide here a link + to documentation that you have for such purposes and also describe how you are going to + understand such files during next release, when you (very likely) slightly change the + format. + </p> + </hint> + </question> +--> + <answer id="format-types"> + <p> + XXX no answer for format-types + </p> + </answer> + + + +<!-- + <question id="lookup-lookup" when="init"> + Does your module use <code>org.openide.util.Lookup</code> + or any similar technology to find any components to communicate with? Which ones? + + <hint> + NetBeans is build around a generic registry of services called + lookup. It is preferable to use it for registration and discovery + if possible. See + <a href="http://www.netbeans.org/download/dev/javadoc/org-openide-util/org/openide/util/lookup/doc-files/index.html" shape="rect"> + The Solution to Comunication Between Components + </a>. If you do not plan to use lookup and insist usage + of other solution, then please describe why it is not working for + you. + <br/> + When filling the final version of your arch document, please + describe the interfaces you are searching for, where + are defined, whether you are searching for just one or more of them, + if the order is important, etc. Also classify the stability of such + API contract. Use <api group=&lookup& /> tag, so + your information gets listed in the summary page of your javadoc. + </hint> + </question> +--> + <answer id="lookup-lookup"> + <p> + None. + </p> + </answer> + + + +<!-- + <question id="lookup-register" when="final"> + Do you register anything into lookup for other code to find? + <hint> + Do you register using layer file or using a declarative annotation such as <code>@ServiceProvider</code>? + Who is supposed to find your component? + </hint> + </question> +--> + <answer id="lookup-register"> + <p> + XXX no answer for lookup-register + </p> + </answer> + + + +<!-- + <question id="lookup-remove" when="final"> + Do you remove entries of other modules from lookup? + <hint> + Why? Of course, that is possible, but it can be dangerous. Is the module + your are masking resource from aware of what you are doing? + </hint> + </question> +--> + <answer id="lookup-remove"> + <p> + XXX no answer for lookup-remove + </p> + </answer> + + + +<!-- + <question id="perf-exit" when="final"> + Does your module run any code on exit? + </question> +--> + <answer id="perf-exit"> + <p> + XXX no answer for perf-exit + </p> + </answer> + + + +<!-- + <question id="perf-huge_dialogs" when="final"> + Does your module contain any dialogs or wizards with a large number of + GUI controls such as combo boxes, lists, trees, or text areas? + </question> +--> + <answer id="perf-huge_dialogs"> + <p> + XXX no answer for perf-huge_dialogs + </p> + </answer> + + + +<!-- + <question id="perf-limit" when="init"> + Are there any hard-coded or practical limits in the number or size of + elements your code can handle? + <hint> + Most of algorithms have increasing memory and speed complexity + with respect to size of data they operate on. What is the critical + part of your project that can be seen as a bottleneck with + respect to speed or required memory? What are the practical + sizes of data you tested your project with? What is your estimate + of potential size of data that would cause visible performance + problems? Is there some kind of check to detect such situation + and prevent "hard" crashes - for example the CloneableEditorSupport + checks for size of a file to be opened in editor + and if it is larger than 1Mb it shows a dialog giving the + user the right to decide - e.g. to cancel or commit suicide. + </hint> + </question> +--> + <answer id="perf-limit"> + <p> + XXX no answer for perf-limit + </p> + </answer> + + + +<!-- + <question id="perf-mem" when="final"> + How much memory does your component consume? Estimate + with a relation to the number of windows, etc. + </question> +--> + <answer id="perf-mem"> + <p> + XXX no answer for perf-mem + </p> + </answer> + + + +<!-- + <question id="perf-menus" when="final"> + Does your module use dynamically updated context menus, or + context-sensitive actions with complicated and slow enablement logic? + <hint> + If you do a lot of tricks when adding actions to regular or context menus, you can significantly + slow down display of the menu, even when the user is not using your action. Pay attention to + actions you add to the main menu bar, and to context menus of foreign nodes or components. If + the action is conditionally enabled, or changes its display dynamically, you need to check the + impact on performance. In some cases it may be more appropriate to make a simple action that is + always enabled but does more detailed checks in a dialog if it is actually run. + </hint> + </question> +--> + <answer id="perf-menus"> + <p> + XXX no answer for perf-menus + </p> + </answer> + + + +<!-- + <question id="perf-progress" when="final"> + Does your module execute any long-running tasks? + + <hint>Long running tasks should never block + AWT thread as it badly hurts the UI + <a href="http://performance.netbeans.org/responsiveness/issues.html" shape="rect"> + responsiveness</a>. + Tasks like connecting over + network, computing huge amount of data, compilation + be done asynchronously (for example + using <code>RequestProcessor</code>), definitively it should + not block AWT thread. + </hint> + </question> +--> + <answer id="perf-progress"> + <p> + XXX no answer for perf-progress + </p> + </answer> + + + +<!-- + <question id="perf-scale" when="init"> + Which external criteria influence the performance of your + program (size of file in editor, number of files in menu, + in source directory, etc.) and how well your code scales? + <hint> + Please include some estimates, there are other more detailed + questions to answer in later phases of implementation. + </hint> + </question> +--> + <answer id="perf-scale"> + <p> + XXX no answer for perf-scale + </p> + </answer> + + + +<!-- + <question id="perf-spi" when="init"> + How the performance of the plugged in code will be enforced? + <hint> + If you allow foreign code to be plugged into your own module, how + do you enforce that it will behave correctly and quickly and will not + negatively influence the performance of your own module? + </hint> + </question> +--> + <answer id="perf-spi"> + <p> + XXX no answer for perf-spi + </p> + </answer> + + + +<!-- + <question id="perf-startup" when="final"> + Does your module run any code on startup? + </question> +--> + <answer id="perf-startup"> + <p> + XXX no answer for perf-startup + </p> + </answer> + + + +<!-- + <question id="perf-wakeup" when="final"> + Does any piece of your code wake up periodically and do something + even when the system is otherwise idle (no user interaction)? + </question> +--> + <answer id="perf-wakeup"> + <p> + XXX no answer for perf-wakeup + </p> + </answer> + + + +<!-- + <question id="resources-layer" when="final"> + Does your module provide own layer? Does it create any files or + folders in it? What it is trying to communicate by that and with which + components? + + <hint> + NetBeans allows automatic and declarative installation of resources + by module layers. Module register files into appropriate places + and other components use that information to perform their task + (build menu, toolbar, window layout, list of templates, set of + options, etc.). + </hint> + </question> +--> + <answer id="resources-layer"> + <p> + XXX no answer for resources-layer + </p> + </answer> + + + +<!-- + <question id="resources-mask" when="final"> + Does your module mask/hide/override any resources provided by other modules in + their layers? + + <hint> + If you mask a file provided by another module, you probably depend + on that and do not want the other module to (for example) change + the file's name. That module shall thus make that file available as an API + of some stability category. + </hint> + </question> +--> + <answer id="resources-mask"> + <p> + XXX no answer for resources-mask + </p> + </answer> + + + +<!-- + <question id="resources-preferences" when="final"> + Does your module uses preferences via Preferences API? Does your module use NbPreferences or + or regular JDK Preferences ? Does it read, write or both ? + Does it share preferences with other modules ? If so, then why ? + <hint> + You may use + <api type="export" group="preferences" + name="preference node name" category="private"> + description of individual keys, where it is used, what it + influences, whether the module reads/write it, etc. + </api> + Due to XML ID restrictions, rather than /org/netbeans/modules/foo give the "name" as org.netbeans.modules.foo. + Note that if you use NbPreferences this name will then be the same as the code name base of the module. + </hint> + </question> +--> + <answer id="resources-preferences"> + <p> + XXX no answer for resources-preferences + </p> + </answer> + + + +<!-- + <question id="security-grant" when="final"> + Does your code grant additional rights to some other code? + <hint>Avoid using a class loader that adds extra + permissions to loaded code unless really necessary. + Also note that your API implementation + can also expose unneeded permissions to enemy code by + calling AccessController.doPrivileged().</hint> + </question> +--> + <answer id="security-grant"> + <p> + XXX no answer for security-grant + </p> + </answer> + + + +<!-- + <question id="security-policy" when="final"> + Does your functionality require modifications to the standard policy file? + <hint>Your code might pass control to third-party code not + coming from trusted domains. This could be code downloaded over the + network or code coming from libraries that are not bundled + with NetBeans. Which permissions need to be granted to which domains?</hint> + </question> +--> + <answer id="security-policy"> + <p> + XXX no answer for security-policy + </p> + </answer> + +</api-answers> + diff --git a/ide/lexer.antlr4/build.xml b/ide/lexer.antlr4/build.xml new file mode 100644 index 0000000000..73dc401b13 --- /dev/null +++ b/ide/lexer.antlr4/build.xml @@ -0,0 +1,41 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<project name="ide/lexer.antlr4" default="build" basedir="."> + <import file="../../nbbuild/templates/projectized.xml"/> + + <target name="generate-test-lexer" description="Generate DummyLexer for tests" depends="init,test-init"> + <property name="test.outdir" location="${test.unit.src.dir}/org/antlr/grammars/dummy"/> + <path id="antlr4.tool"> + <fileset dir="../../ide/libs.antlr4.runtime/external" includes="*.jar"/> + <fileset dir="../../ide/libs.antlr3.runtime/external" includes="antlr-runtime-*.jar"/> + </path> + + <java classname="org.antlr.v4.Tool" classpathref="antlr4.tool" fork="true" dir="${test.outdir}"> + <arg value="-o"/> + <arg value="${test.outdir}"/> + <arg value="DummyLexer.g4"/> + </java> + + <delete dir="${test.outdir}" includes="*.tokens"/> + <delete dir="${test.outdir}" includes="*.interp"/> + </target> +</project> diff --git a/ide/lexer.antlr4/manifest.mf b/ide/lexer.antlr4/manifest.mf new file mode 100644 index 0000000000..52b3d99e74 --- /dev/null +++ b/ide/lexer.antlr4/manifest.mf @@ -0,0 +1,3 @@ +OpenIDE-Module: org.netbeans.modules.lexer.antlr4 +OpenIDE-Module-Implementation-Version: 1 +OpenIDE-Module-Localizing-Bundle: org/netbeans/spi/lexer/antlr4/Bundle.properties diff --git a/ide/lexer.antlr4/nbproject/project.properties b/ide/lexer.antlr4/nbproject/project.properties new file mode 100644 index 0000000000..0c48343d27 --- /dev/null +++ b/ide/lexer.antlr4/nbproject/project.properties @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +is.autoload=true +javac.compilerargs=-Xlint:unchecked +javac.source=1.8 +spec.version.base=1.0.0 + diff --git a/ide/lexer.antlr4/nbproject/project.xml b/ide/lexer.antlr4/nbproject/project.xml new file mode 100644 index 0000000000..9d9e528324 --- /dev/null +++ b/ide/lexer.antlr4/nbproject/project.xml @@ -0,0 +1,61 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<project xmlns="http://www.netbeans.org/ns/project/1"> + <type>org.netbeans.modules.apisupport.project</type> + <configuration> + <data xmlns="http://www.netbeans.org/ns/nb-module-project/3"> + <code-name-base>org.netbeans.modules.lexer.antlr4</code-name-base> + <module-dependencies> + <dependency> + <code-name-base>org.netbeans.libs.antlr4.runtime</code-name-base> + <build-prerequisite/> + <compile-dependency/> + <run-dependency> + <release-version>2</release-version> + <specification-version>1.20</specification-version> + </run-dependency> + </dependency> + <dependency> + <code-name-base>org.netbeans.modules.lexer</code-name-base> + <build-prerequisite/> + <compile-dependency/> + <run-dependency> + <release-version>2</release-version> + <specification-version>1.81</specification-version> + </run-dependency> + </dependency> + </module-dependencies> + <test-dependencies> + <test-type> + <name>unit</name> + <test-dependency> + <code-name-base>org.netbeans.libs.junit4</code-name-base> + <compile-dependency/> + </test-dependency> + </test-type> + </test-dependencies> + <public-packages> + <package>org.netbeans.spi.lexer.antlr4</package> + </public-packages> + </data> + </configuration> +</project> diff --git a/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/AbstractAntlrLexerBridge.java b/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/AbstractAntlrLexerBridge.java new file mode 100644 index 0000000000..112936af59 --- /dev/null +++ b/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/AbstractAntlrLexerBridge.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.netbeans.spi.lexer.antlr4; + +import org.antlr.v4.runtime.misc.IntegerList; +import org.netbeans.api.lexer.Token; +import org.antlr.v4.runtime.Lexer; +import static org.antlr.v4.runtime.Recognizer.EOF; +import org.netbeans.api.lexer.TokenId; +import org.netbeans.spi.lexer.LexerRestartInfo; +import org.netbeans.spi.lexer.TokenFactory; + +/** + * Implementations of this class shall take an ANTLRv4 Lexer and map ANTLR tokens + * to NetBeans Lexer tokens. + * + * @author Laszlo Kishalmi + * @param <L> The ANTLR Lexer type + * @param <T> The NetBeans TokenId type + */ +public abstract class AbstractAntlrLexerBridge<L extends Lexer, T extends TokenId> implements org.netbeans.spi.lexer.Lexer<T> { + + private final TokenFactory<T> tokenFactory; + protected final L lexer; + protected final LexerInputCharStream input; + + /** + * Constructor for the lexer bridge, usually used as: + * <pre>{@code + * public SomeLexer(LexerRestartInfo<SomeTokenId> info) { + * super(info, new SomeANTLRLexer(new LexerInputCharStream(info.input()))); + * } + * } + * </pre> + * @param info The lexer restart info + * @param lexer The ANTLR generated Lexer + */ + @SuppressWarnings("unchecked") + public AbstractAntlrLexerBridge(LexerRestartInfo<T> info, L lexer) { + this.tokenFactory = info.tokenFactory(); + this.lexer = lexer; + this.input = (LexerInputCharStream) lexer.getInputStream(); + if (info.state() != null) { + ((LexerState<L>) info.state()).restore(lexer); + } + input.markToken(); + } + + + private org.antlr.v4.runtime.Token preFetchedToken = null; + + @Override + public final Token<T> nextToken() { + org.antlr.v4.runtime.Token nextToken; + if (preFetchedToken != null) { + nextToken = preFetchedToken; + input.seek(preFetchedToken.getStopIndex() + 1); + preFetchedToken = null; + } else { + nextToken = nextRealToken(); + } + return nextToken.getType() != EOF ? mapToken(nextToken.getType()) : null; + } + + /** + * Implementations shall provide a suitable mapping between ANTLR lexer + * token types and NetBeans lexer tokens. The mapping is usually many to one, + * could be implemented as: + * <pre>{@code + * switch (antlrTokenType) { + * case DOC_COMMENT: + * case BLOCK_COMMENT: + * case LINE_COMMENT: + * return token(SomeTokenId.COMMENT); +* + * case STRING_CHAR: + * return groupToken(SomeTokenId.STRING); + * default: + * return token(SomeTokenId.ERROR); + * } + * }</pre> + * @param antlrTokenType the token type from the ANTLR Lexer + * + * @return a NetBeans lexer token. + */ + protected abstract Token<T> mapToken(int antlrTokenType); + + @Override + /** + * This method can be overridden, if some resource cleanup is needed when + * the lexer is no longer in use. Usually not required for ANTLR lexers. + */ + public void release() { + } + + @Override + public Object state() { + return new LexerState<>(lexer); + } + + /** + * Some grammars provide big chunks of the same token. It is recommended to + * group those tokens together in one Token for the editor for performance + * reasons. + * + * @param id the NetBeans TokenId of the returned token. + * @param antlrTokenType the ANTLR token type to be collected till found. + * + * @return one lexer token, that represents several ANTLR tokens. + */ + protected final Token<T> groupToken(T id, int antlrTokenType) { + preFetchedToken = nextRealToken(); + while (preFetchedToken.getType() == antlrTokenType) { + preFetchedToken = nextRealToken(); + } + input.seek(preFetchedToken.getStartIndex()); + return token(id); + } + + protected final Token<T> token(T id) { + input.markToken(); + return tokenFactory.createToken(id); + } + + private org.antlr.v4.runtime.Token nextRealToken() { + int index = input.index(); + org.antlr.v4.runtime.Token ret = lexer.nextToken(); + // Filtering out pseudo tokens + while ((ret.getType() != EOF) && (index == input.index())) { + ret = lexer.nextToken(); + } + return ret; + } + + /** + * This class can save and restore the internal state of a basic ANTLR Lexer. + * Some grammars has additional state information, for those this class + * needs to be extended, and the {@link #state()} method has to be + * overridden. + * + * @param <L> The ANTLR Lexer, that state is to be kept. + */ + public static class LexerState<L extends Lexer> { + final int state; + final int mode; + final IntegerList modes; + + public LexerState(L lexer) { + this.state= lexer.getState(); + + this.mode = lexer._mode; + this.modes = new IntegerList(lexer._modeStack); + } + + public void restore(L lexer) { + lexer.setState(state); + lexer._modeStack.addAll(modes); + lexer._mode = mode; + } + + @Override + public String toString() { + return String.valueOf(state); + } + } + +} diff --git a/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/AntlrTokenSequence.java b/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/AntlrTokenSequence.java new file mode 100644 index 0000000000..cf7eb0010e --- /dev/null +++ b/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/AntlrTokenSequence.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.netbeans.spi.lexer.antlr4; + +import java.util.ArrayList; +import java.util.ListIterator; +import java.util.Optional; +import java.util.function.Predicate; +import org.antlr.v4.runtime.Lexer; +import static org.antlr.v4.runtime.Recognizer.EOF; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenSource; + +/** + * This class provides sequential cursor based access to ANTLR Lexer tokens. + * The cursor can be sought in the input stream. + * + * @author lkishalmi + */ +public final class AntlrTokenSequence { + + private final TokenSource tokens; + private boolean eofRead = false; + private int readIndex; + private int cursorOffset; + private ListIterator<Token> cursor; + + private final ArrayList<Token> tokenList = new ArrayList<>(200); + + /** Predicate constant to select tokens on the DEFAULT_CHANNEL */ + public static final Predicate<Token> DEFAULT_CHANNEL = new ChannelFilter(Lexer.DEFAULT_TOKEN_CHANNEL); + + /** Predicate class to filter tokens by a selected channel */ + public static final class ChannelFilter implements Predicate<Token> { + private final int channel; + + public ChannelFilter(int channel) { + this.channel = channel; + } + + @Override + public boolean test(Token t) { + return channel == t.getChannel(); + } + } + + /** + * Creates a AntlrTokenSequence over the provided TokenSource (Lexer) + * + * @param tokens the token source to be read. + */ + public AntlrTokenSequence(TokenSource tokens) { + this.tokens = tokens; + this.cursor = tokenList.listIterator(); + } + + /** + * Seeks the cursor to an offset in the input stream. If the offset is + * inside a token, then the {@linkplain #next()} method would return the + * token which contains the offset. + * + * @param offset the position in the lexer input stream to seek to. + */ + public void seekTo(int offset) { + if (offset > readIndex) { + if (cursor.hasNext()) { + //replace the cursor if it is not at the end of the list. + cursor = tokenList.listIterator(tokenList.size()); + } + Token t = read(); + while ((t != null) && (t.getStopIndex() + 1 < offset)) { + t = read(); + } + if (t != null && (offset < t.getStopIndex() + 1)) { + cursor.previous(); + cursorOffset = t.getStartIndex(); + } else { + cursorOffset = readIndex; + } + } else { + if (offset > getOffset()) { + next((t) -> t.getStopIndex() > offset); + if (cursor.hasPrevious()) { + cursor.previous(); + } + + } else { + previous((t) -> t.getStartIndex() < offset); + } + } + + } + + /** + * Returns {@code true} if the lexer stream is empty. + * + * @return {@code true} if the lexer stream is empty. + */ + public boolean isEmpty() { + return tokenList.isEmpty() && !hasNext(); + } + + /** + * Returns {@code true} if there is a token to read after the cursor. + * + * @return {@code true} if there is a token to read after the cursor. + */ + public boolean hasNext() { + if (!eofRead && (cursorOffset == readIndex) && !cursor.hasNext()) { + Token t = read(); + if (t != null) { + cursor.previous(); + } + } + return !(eofRead && !cursor.hasNext()); + } + + /** + * Returns {@code true} if there is a token to read before the cursor. + * + * @return {@code true} if there is a token to read before the cursor. + */ + public boolean hasPrevious() { + return cursor.hasPrevious(); + } + + /** + * Returns the offset of the cursor. It always returns index on token + * boundary. + * + * @return the offset of the cursor. + */ + public int getOffset() { + return cursorOffset; + } + + /** + * Returns the token before the cursor if there is one. It also moves the + * cursor left by one. + * + * @return the token before the cursor. + */ + public Optional<Token> previous() { + Optional<Token> ret = cursor.hasPrevious() ? Optional.of(cursor.previous()) : Optional.empty(); + cursorOffset = cursor.hasPrevious() ? ret.get().getStartIndex() : 0; + return ret; + + } + + /** + * Returns the token before the cursor that satisfies the provided predicate + * condition. It also moves the cursor at the start of the returned token. + * + * @param filter the predicate to satisfy. + * @return the token before the cursor. + */ + public Optional<Token> previous(Predicate<Token> filter) { + Optional<Token> ot = previous(); + while (ot.isPresent() && !filter.test(ot.get())) { + ot = previous(); + } + return ot; + } + + /** + * Returns the token before the cursor that matches the provided token type. + * It also moves the cursor at the start of the returned token. + * + * @param tokenType the searched token type + * @return the token before the cursor. + */ + public Optional<Token> previous(int tokenType){ + return previous((Token t) -> t.getType() == tokenType); + } + + /** + * Returns the token after the cursor if there is one. It also moves the + * cursor right by one. + * + * @return the token after the cursor. + */ + public Optional<Token> next() { + if (hasNext()) { + Token t = cursor.next(); + cursorOffset = t.getStopIndex() + 1; + return Optional.of(t); + } else { + return Optional.empty(); + } + } + + /** + * Returns the token after the cursor that satisfies the provided predicate + * condition. It also moves the cursor at the end of the returned token. + * + * @param filter the predicate to satisfy. + * @return the token after the cursor. + */ + public Optional<Token> next(Predicate<Token> filter) { + Optional<Token> ot = next(); + while (ot.isPresent() && !filter.test(ot.get())) { + ot = next(); + } + return ot; + } + + /** + * Returns the token after the cursor that matches the provided token type. + * It also moves the cursor at the end of the returned token. + * + * @param tokenType the searched token type + * @return the token after the cursor. + */ + public Optional<Token> next(int tokenType){ + return next((Token t) -> t.getType() == tokenType); + } + + private Token read() { + if (eofRead) { + return null; + } + Token t = tokens.nextToken(); + if (t.getType() != EOF) { + cursor.add(t); + readIndex = t.getStopIndex() + 1; + return t; + } else { + eofRead = true; + return null; + } + } + +} diff --git a/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/Bundle.properties b/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/Bundle.properties new file mode 100644 index 0000000000..bf38471c31 --- /dev/null +++ b/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/Bundle.properties @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +OpenIDE-Module-Name=ANTLR4 Lexer Support +OpenIDE-Module-Display-Category=Editing +OpenIDE-Module-Short-Description=Provides Utility classes for ANTLR4 Lexer Integration +OpenIDE-Module-Long-Description=This module provides some utility classes \ +that make an ANTLR4 based Lexer easier to integrate in the IDE. + diff --git a/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/LexerInputCharStream.java b/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/LexerInputCharStream.java new file mode 100644 index 0000000000..a0853ba63a --- /dev/null +++ b/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/LexerInputCharStream.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.netbeans.spi.lexer.antlr4; + +import org.antlr.v4.runtime.CharStream; +import static org.antlr.v4.runtime.IntStream.UNKNOWN_SOURCE_NAME; +import org.antlr.v4.runtime.misc.Interval; +import org.netbeans.spi.lexer.*; + +/** + * An ANTLR4 CharSteam implementation over NetBeans LexerInput. + * + * @author Laszlo Kishalmi + */ +public class LexerInputCharStream implements CharStream { + private final LexerInput input; + + private int tokenMark = Integer.MAX_VALUE; + private int index = 0; + + public LexerInputCharStream(LexerInput input) { + this.input = input; + } + + @Override + public String getText(Interval intrvl) { + if (intrvl.a < tokenMark) { + throw new UnsupportedOperationException("Can't read before the last token end: " + tokenMark); + } + int start = intrvl.a - tokenMark; + int end = intrvl.b - tokenMark + 1; + int toread = end - start - input.readLength(); + for (int i = 0; i < toread; i++) { + input.read(); + } + String ret = String.valueOf(input.readText(start, end)); + if (toread > 0) { + input.backup(toread); + } + return ret; + } + + @Override + public void consume() { + read(); + } + + @Override + public int LA(int count) { + if (count == 0) { + return 0; //the behaviour is not defined + } + + int c = 0; + if (count > 0) { + for (int i = 0; i < count; i++) { + c = read(); + } + backup(count); + } else { + backup(count); + c = read(); + } + return c; + } + + //Marks are for buffering in ANTLR4, we do not really need them + @Override + public int mark() { + return -1; + } + + public void markToken() { + tokenMark = index; + } + + @Override + public void release(int marker) { + } + + @Override + public int index() { + return index; + } + + @Override + public void seek(int i) { + if (i < index()) { + backup(index() - i); + } else { + while (index() < i) { + if (read() == LexerInput.EOF) { + break; + } + } + } + } + + + private int read() { + int ret = input.read(); + index += 1; + return ret; + } + + private void backup(int count) { + index -= count; + input.backup(count); + } + + @Override + public int size() { + throw new UnsupportedOperationException("Stream size is unknown."); + } + + @Override + public String getSourceName() { + return UNKNOWN_SOURCE_NAME; + } +} diff --git a/ide/lexer.antlr4/test/unit/src/org/antlr/grammars/dummy/DummyLexer.g4 b/ide/lexer.antlr4/test/unit/src/org/antlr/grammars/dummy/DummyLexer.g4 new file mode 100644 index 0000000000..7f4e0ff039 --- /dev/null +++ b/ide/lexer.antlr4/test/unit/src/org/antlr/grammars/dummy/DummyLexer.g4 @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +lexer grammar DummyLexer; + +@header{ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// DO NOT EDIT THIS FILE MANUALLY! +// SEE build.xml FOR INSTRUCTIONS +package org.antlr.grammars.dummy; + +} +channels {COMMENT} +BLOCK_COMMENT + : '/*' .*? ('*/' | EOF) -> channel(COMMENT) + ; + +WS + : [ \t\r\n]+ -> channel(HIDDEN) + ; + +// Let's have a keyword +LEXER + : 'lexer' + ; + +// Cover some random text +ID + : [a-zA-Z0-9\\.-]+ + ; + +ERROR + : '.' -> skip + ; \ No newline at end of file diff --git a/ide/lexer.antlr4/test/unit/src/org/antlr/grammars/dummy/DummyLexer.java b/ide/lexer.antlr4/test/unit/src/org/antlr/grammars/dummy/DummyLexer.java new file mode 100644 index 0000000000..a0071db720 --- /dev/null +++ b/ide/lexer.antlr4/test/unit/src/org/antlr/grammars/dummy/DummyLexer.java @@ -0,0 +1,175 @@ +// Generated from java-escape by ANTLR 4.11.1 + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// DO NOT EDIT THIS FILE MANUALLY! +// SEE build.xml FOR INSTRUCTIONS +package org.antlr.grammars.dummy; + + +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.misc.*; + +@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue"}) +public class DummyLexer extends Lexer { + static { RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION); } + + protected static final DFA[] _decisionToDFA; + protected static final PredictionContextCache _sharedContextCache = + new PredictionContextCache(); + public static final int + BLOCK_COMMENT=1, WS=2, LEXER=3, ID=4, ERROR=5; + public static final int + COMMENT=2; + public static String[] channelNames = { + "DEFAULT_TOKEN_CHANNEL", "HIDDEN", "COMMENT" + }; + + public static String[] modeNames = { + "DEFAULT_MODE" + }; + + private static String[] makeRuleNames() { + return new String[] { + "BLOCK_COMMENT", "WS", "LEXER", "ID", "ERROR" + }; + } + public static final String[] ruleNames = makeRuleNames(); + + private static String[] makeLiteralNames() { + return new String[] { + null, null, null, "'lexer'", null, "'.'" + }; + } + private static final String[] _LITERAL_NAMES = makeLiteralNames(); + private static String[] makeSymbolicNames() { + return new String[] { + null, "BLOCK_COMMENT", "WS", "LEXER", "ID", "ERROR" + }; + } + private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames(); + public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); + + /** + * @deprecated Use {@link #VOCABULARY} instead. + */ + @Deprecated + public static final String[] tokenNames; + static { + tokenNames = new String[_SYMBOLIC_NAMES.length]; + for (int i = 0; i < tokenNames.length; i++) { + tokenNames[i] = VOCABULARY.getLiteralName(i); + if (tokenNames[i] == null) { + tokenNames[i] = VOCABULARY.getSymbolicName(i); + } + + if (tokenNames[i] == null) { + tokenNames[i] = "<INVALID>"; + } + } + } + + @Override + @Deprecated + public String[] getTokenNames() { + return tokenNames; + } + + @Override + + public Vocabulary getVocabulary() { + return VOCABULARY; + } + + + public DummyLexer(CharStream input) { + super(input); + _interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache); + } + + @Override + public String getGrammarFileName() { return "DummyLexer.g4"; } + + @Override + public String[] getRuleNames() { return ruleNames; } + + @Override + public String getSerializedATN() { return _serializedATN; } + + @Override + public String[] getChannelNames() { return channelNames; } + + @Override + public String[] getModeNames() { return modeNames; } + + @Override + public ATN getATN() { return _ATN; } + + public static final String _serializedATN = + "\u0004\u0000\u00051\u0006\uffff\uffff\u0002\u0000\u0007\u0000\u0002\u0001"+ + "\u0007\u0001\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004"+ + "\u0007\u0004\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0005\u0000"+ + "\u0010\b\u0000\n\u0000\f\u0000\u0013\t\u0000\u0001\u0000\u0001\u0000\u0001"+ + "\u0000\u0003\u0000\u0018\b\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0004"+ + "\u0001\u001d\b\u0001\u000b\u0001\f\u0001\u001e\u0001\u0001\u0001\u0001"+ + "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"+ + "\u0001\u0003\u0004\u0003*\b\u0003\u000b\u0003\f\u0003+\u0001\u0004\u0001"+ + "\u0004\u0001\u0004\u0001\u0004\u0001\u0011\u0000\u0005\u0001\u0001\u0003"+ + "\u0002\u0005\u0003\u0007\u0004\t\u0005\u0001\u0000\u0002\u0003\u0000\t"+ + "\n\r\r \u0005\u0000-.09AZ\\\\az4\u0000\u0001\u0001\u0000\u0000\u0000"+ + "\u0000\u0003\u0001\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000"+ + "\u0000\u0007\u0001\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0001"+ + "\u000b\u0001\u0000\u0000\u0000\u0003\u001c\u0001\u0000\u0000\u0000\u0005"+ + "\"\u0001\u0000\u0000\u0000\u0007)\u0001\u0000\u0000\u0000\t-\u0001\u0000"+ + "\u0000\u0000\u000b\f\u0005/\u0000\u0000\f\r\u0005*\u0000\u0000\r\u0011"+ + "\u0001\u0000\u0000\u0000\u000e\u0010\t\u0000\u0000\u0000\u000f\u000e\u0001"+ + "\u0000\u0000\u0000\u0010\u0013\u0001\u0000\u0000\u0000\u0011\u0012\u0001"+ + "\u0000\u0000\u0000\u0011\u000f\u0001\u0000\u0000\u0000\u0012\u0017\u0001"+ + "\u0000\u0000\u0000\u0013\u0011\u0001\u0000\u0000\u0000\u0014\u0015\u0005"+ + "*\u0000\u0000\u0015\u0018\u0005/\u0000\u0000\u0016\u0018\u0005\u0000\u0000"+ + "\u0001\u0017\u0014\u0001\u0000\u0000\u0000\u0017\u0016\u0001\u0000\u0000"+ + "\u0000\u0018\u0019\u0001\u0000\u0000\u0000\u0019\u001a\u0006\u0000\u0000"+ + "\u0000\u001a\u0002\u0001\u0000\u0000\u0000\u001b\u001d\u0007\u0000\u0000"+ + "\u0000\u001c\u001b\u0001\u0000\u0000\u0000\u001d\u001e\u0001\u0000\u0000"+ + "\u0000\u001e\u001c\u0001\u0000\u0000\u0000\u001e\u001f\u0001\u0000\u0000"+ + "\u0000\u001f \u0001\u0000\u0000\u0000 !\u0006\u0001\u0001\u0000!\u0004"+ + "\u0001\u0000\u0000\u0000\"#\u0005l\u0000\u0000#$\u0005e\u0000\u0000$%"+ + "\u0005x\u0000\u0000%&\u0005e\u0000\u0000&\'\u0005r\u0000\u0000\'\u0006"+ + "\u0001\u0000\u0000\u0000(*\u0007\u0001\u0000\u0000)(\u0001\u0000\u0000"+ + "\u0000*+\u0001\u0000\u0000\u0000+)\u0001\u0000\u0000\u0000+,\u0001\u0000"+ + "\u0000\u0000,\b\u0001\u0000\u0000\u0000-.\u0005.\u0000\u0000./\u0001\u0000"+ + "\u0000\u0000/0\u0006\u0004\u0002\u00000\n\u0001\u0000\u0000\u0000\u0005"+ + "\u0000\u0011\u0017\u001e+\u0003\u0000\u0002\u0000\u0000\u0001\u0000\u0006"+ + "\u0000\u0000"; + public static final ATN _ATN = + new ATNDeserializer().deserialize(_serializedATN.toCharArray()); + static { + _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()]; + for (int i = 0; i < _ATN.getNumberOfDecisions(); i++) { + _decisionToDFA[i] = new DFA(_ATN.getDecisionState(i), i); + } + } +} \ No newline at end of file diff --git a/java/languages.antlr/test/unit/src/org/netbeans/modules/languages/antlr/AntlrTokenSequenceTest.java b/ide/lexer.antlr4/test/unit/src/org/netbeans/spi/lexer/antlr4/AntlrTokenSequenceTest.java similarity index 90% copy from java/languages.antlr/test/unit/src/org/netbeans/modules/languages/antlr/AntlrTokenSequenceTest.java copy to ide/lexer.antlr4/test/unit/src/org/netbeans/spi/lexer/antlr4/AntlrTokenSequenceTest.java index 1e3f775a5a..c3c9f674bc 100644 --- a/java/languages.antlr/test/unit/src/org/netbeans/modules/languages/antlr/AntlrTokenSequenceTest.java +++ b/ide/lexer.antlr4/test/unit/src/org/netbeans/spi/lexer/antlr4/AntlrTokenSequenceTest.java @@ -16,23 +16,23 @@ * specific language governing permissions and limitations * under the License. */ -package org.netbeans.modules.languages.antlr; +package org.netbeans.spi.lexer.antlr4; -import org.antlr.parser.antlr4.ANTLRv4Lexer; +import org.antlr.grammars.dummy.DummyLexer; import org.antlr.v4.runtime.CharStreams; import org.junit.Test; - import static org.junit.Assert.*; -import static org.netbeans.modules.languages.antlr.AntlrTokenSequence.DEFAULT_CHANNEL; +import static org.netbeans.spi.lexer.antlr4.AntlrTokenSequence.DEFAULT_CHANNEL; + /** * - * @author lkishalmi + * @author Laszlo Kishalmi */ public class AntlrTokenSequenceTest { - + public AntlrTokenSequenceTest() { } - + /** * Test of seekTo method, of class AntlrTokenSequence. */ @@ -49,7 +49,7 @@ public class AntlrTokenSequenceTest { public void testSeekTo2() { System.out.println("seekTo"); int offset = 0; - AntlrTokenSequence instance = sequence("/**/"); + AntlrTokenSequence instance = sequence("# Properties"); instance.seekTo(offset); assertFalse(instance.isEmpty()); assertTrue(instance.next().isPresent()); @@ -139,9 +139,9 @@ public class AntlrTokenSequenceTest { @Test public void testGetOffset1() { AntlrTokenSequence instance = sequence("/* */lexer"); - instance.seekTo(7); + instance.seekTo("/* */le".length()); assertTrue(instance.hasNext()); - assertEquals(5, instance.getOffset()); + assertEquals("/* */".length(), instance.getOffset()); instance.previous(); assertEquals(0, instance.getOffset()); } @@ -152,7 +152,7 @@ public class AntlrTokenSequenceTest { assertTrue(instance.hasNext()); instance.next(DEFAULT_CHANNEL).ifPresent((t) -> assertEquals("lexer", t.getText())); instance.next(DEFAULT_CHANNEL).ifPresent((t) -> assertEquals("grammar", t.getText())); - assertFalse(instance.hasNext()); + assertFalse(instance.hasNext()); } @Test @@ -162,10 +162,10 @@ public class AntlrTokenSequenceTest { assertFalse(instance.hasNext()); instance.previous(DEFAULT_CHANNEL).ifPresent((t) -> assertEquals("grammar", t.getText())); instance.previous(DEFAULT_CHANNEL).ifPresent((t) -> assertEquals("lexer", t.getText())); - assertTrue(instance.hasPrevious()); + assertTrue(instance.hasPrevious()); } private AntlrTokenSequence sequence(String s) { - return new AntlrTokenSequence(new ANTLRv4Lexer(CharStreams.fromString(s))); + return new AntlrTokenSequence(new DummyLexer(CharStreams.fromString(s))); } } diff --git a/java/languages.antlr/test/unit/src/org/netbeans/modules/languages/antlr/AntlrTokenSequenceTest.java b/java/languages.antlr/test/unit/src/org/netbeans/modules/languages/antlr/AntlrTokenSequenceTest.java index 1e3f775a5a..216a918201 100644 --- a/java/languages.antlr/test/unit/src/org/netbeans/modules/languages/antlr/AntlrTokenSequenceTest.java +++ b/java/languages.antlr/test/unit/src/org/netbeans/modules/languages/antlr/AntlrTokenSequenceTest.java @@ -38,7 +38,6 @@ public class AntlrTokenSequenceTest { */ @Test public void testSeekTo1() { - System.out.println("seekTo"); int offset = 0; AntlrTokenSequence instance = sequence(""); instance.seekTo(offset); @@ -47,7 +46,6 @@ public class AntlrTokenSequenceTest { @Test public void testSeekTo2() { - System.out.println("seekTo"); int offset = 0; AntlrTokenSequence instance = sequence("/**/"); instance.seekTo(offset); @@ -57,7 +55,6 @@ public class AntlrTokenSequenceTest { @Test public void testSeekTo3() { - System.out.println("seekTo"); int offset = 4; AntlrTokenSequence instance = sequence("/**/"); instance.seekTo(offset); @@ -68,7 +65,6 @@ public class AntlrTokenSequenceTest { @Test public void testSeekTo4() { - System.out.println("seekTo"); int offset = 5; AntlrTokenSequence instance = sequence("/* */lexer"); instance.seekTo(offset); @@ -79,7 +75,6 @@ public class AntlrTokenSequenceTest { @Test public void testSeekTo5() { - System.out.println("seekTo"); AntlrTokenSequence instance = sequence("/* */lexer"); instance.seekTo(10); assertFalse(instance.next().isPresent()); @@ -95,28 +90,24 @@ public class AntlrTokenSequenceTest { */ @Test public void testIsEmpty() { - System.out.println("isEmpty"); AntlrTokenSequence instance = sequence(""); assertTrue(instance.isEmpty()); } @Test public void testHasNext1() { - System.out.println("hasNext"); AntlrTokenSequence instance = sequence("lexer"); assertTrue(instance.hasNext()); } @Test public void testHasNext2() { - System.out.println("hasNext"); AntlrTokenSequence instance = sequence(""); assertFalse(instance.hasNext()); } @Test public void testHasNext3() { - System.out.println("hasNext"); AntlrTokenSequence instance = sequence("lexer"); assertTrue(instance.hasNext()); instance.next(); diff --git a/nbbuild/cluster.properties b/nbbuild/cluster.properties index c5164b6bde..a3946b37c8 100644 --- a/nbbuild/cluster.properties +++ b/nbbuild/cluster.properties @@ -430,6 +430,7 @@ nb.cluster.ide=\ languages.toml,\ languages.yaml,\ lexer,\ + lexer.antlr4,\ lexer.nbbridge,\ lib.terminalemulator,\ libs.antlr3.runtime,\ --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@netbeans.apache.org For additional commands, e-mail: commits-h...@netbeans.apache.org For further information about the NetBeans mailing lists, visit: https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists