This is an automated email from the ASF dual-hosted git repository.
lkishalmi pushed a commit to branch use-snakeyaml-parser-improved
in repository https://gitbox.apache.org/repos/asf/netbeans.git
commit cae021baef6c203a6951af77d2aa517fffd0e333
Author: Laszlo Kishalmi <laszlo.kisha...@gmail.com>
AuthorDate: Tue Oct 12 11:44:08 2021 -0700
Made the new parser work against the tests
---
.../modules/languages/yaml/YamlParser.java | 167 +++++++++--------
.../modules/languages/yaml/YamlParserResult.java | 34 +---
.../modules/languages/yaml/YamlScanner.java | 183 +++----------------
.../modules/languages/yaml/YamlSection.java | 187 +++++++++++++++++++
.../languages/yaml/YamlSemanticAnalyzer.java | 67 +++----
.../modules/languages/yaml/YamlStructureItem.java | 200 +++++++++++++++++++++
.../test/unit/data/testfiles/error.yaml.errors | 2 +-
.../test/unit/data/testfiles/error2.yaml.errors | 2 +-
.../test/unit/data/testfiles/error3.yaml.errors | 2 +-
.../unit/data/testfiles/fixture3.yml.structure | 2 -
.../test/unit/data/testfiles/test5.yaml.structure | 12 +-
.../modules/languages/yaml/YamlParserTest.java | 44 +++--
12 files changed, 553 insertions(+), 349 deletions(-)
diff --git
a/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlParser.java
b/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlParser.java
index 86db263..01bef63 100644
--- a/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlParser.java
+++ b/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlParser.java
@@ -18,8 +18,8 @@
*/
package org.netbeans.modules.languages.yaml;
-import java.util.ArrayList;
-import java.util.Collections;
+import java.util.Deque;
+import java.util.LinkedList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -31,26 +31,23 @@ import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.api.lexer.TokenSequence;
import org.netbeans.modules.csl.api.Severity;
+import org.netbeans.modules.csl.api.StructureItem;
import org.netbeans.modules.csl.spi.DefaultError;
import org.netbeans.modules.parsing.api.Snapshot;
import org.netbeans.modules.parsing.api.Task;
import org.netbeans.modules.parsing.spi.ParseException;
-import org.netbeans.modules.parsing.spi.Parser;
import org.netbeans.modules.parsing.spi.SourceModificationEvent;
import org.openide.util.NbBundle;
-import org.snakeyaml.engine.v2.api.LoadSettings;
-import org.snakeyaml.engine.v2.composer.Composer;
-import org.snakeyaml.engine.v2.nodes.Node;
-import org.snakeyaml.engine.v2.parser.ParserImpl;
-import org.snakeyaml.engine.v2.scanner.ScannerImpl;
-import org.snakeyaml.engine.v2.scanner.StreamReader;
+import org.snakeyaml.engine.v2.exceptions.MarkedYamlEngineException;
+import org.snakeyaml.engine.v2.exceptions.ParserException;
+import org.snakeyaml.engine.v2.exceptions.ScannerException;
/**
* Parser for YAML. Delegates to the YAML parser shipped with JRuby (jvyamlb)
*
* @author Tor Norbye
*/
-public class YamlParser extends Parser {
+public class YamlParser extends org.netbeans.modules.parsing.spi.Parser {
private static final Logger LOGGER =
Logger.getLogger(YamlParser.class.getName());
/**
@@ -93,7 +90,7 @@ public class YamlParser extends Parser {
}
private YamlParserResult resultForTooLargeFile(Snapshot snapshot) {
- YamlParserResult result = new
YamlParserResult(Collections.<Node>emptyList(), this, snapshot, false);
+ YamlParserResult result = new YamlParserResult(snapshot, false);
// FIXME this can violate contract of DefaultError (null fo)
DefaultError error = new DefaultError(null,
NbBundle.getMessage(YamlParser.class, "TooLarge"), null,
snapshot.getSource().getFileObject(), 0, 0, Severity.WARNING);
@@ -102,68 +99,56 @@ public class YamlParser extends Parser {
}
// package private for unit tests
- static String replacePhpFragments(String source) {
- // this is a hack. The right solution would be to create a toplevel
language, which
- // will have embeded yaml and php.
- // This code replaces php fragments with space, because jruby parser
fails
- // on this.
- int startReplace = source.indexOf("<?");
+ static void replaceWithSpaces(StringBuilder source, String startToken,
String endToken) {
+ int startReplace = source.indexOf(startToken);
if (startReplace == -1) {
- return source;
+ return;
}
- StringBuilder result = new StringBuilder(source);
-
while (startReplace > -1) {
- int endReplace = result.indexOf("?>", startReplace);
- if (endReplace > -1) {
- endReplace = endReplace + 1;
- StringBuilder spaces = new StringBuilder(endReplace -
startReplace);
- for (int i = 0; i <= endReplace - startReplace; i++) {
- spaces.append(' ');
+ int endReplace = source.indexOf(endToken, startReplace) + 1;
+ if (endReplace > startReplace) {
+ for (int i = startReplace; i <= endReplace; i++) {
+ source.setCharAt(i, ' ');
}
- result.replace(startReplace, endReplace + 1,
spaces.toString());
- startReplace = result.indexOf("<?", endReplace);
+ startReplace = source.indexOf(startToken, endReplace);
} else {
startReplace = -1;
}
}
- return result.toString();
+ }
+
+ static void replacePhpFragments(StringBuilder source) {
+ // this is a hack. The right solution would be to create a toplevel
language, which
+ // will have embeded yaml and php.
+ // This code replaces php fragments with space, because jruby parser
fails
+ // on this.
+ replaceWithSpaces(source, "<?", "?>");
}
- static String replaceMustache(String source) {
+ static void replaceMustache(StringBuilder source) {
// this is a hack. The right solution would be to create a toplevel
language, which
// will have embeded yaml and something.
// This code replaces mouthstache fragments with space.
- int startReplace = source.indexOf("{{");
- if (startReplace == -1) {
- return source;
- }
+ replaceWithSpaces(source, "{{", "}}");
+ }
- StringBuilder result = new StringBuilder(source);
+ final static Pattern[] SPEC_PATTERN_REPLACEMENTS = new Pattern[]{
+ Pattern.compile("@"),
+ Pattern.compile("\\?"),
+ Pattern.compile("!(?!(omap|!omap))"),};
- while (startReplace > -1) {
- int endReplace = result.indexOf("}}", startReplace);
- if (endReplace > -1) {
- endReplace = endReplace + 1;
- StringBuilder spaces = new StringBuilder(endReplace -
startReplace);
- for (int i = 0; i <= endReplace - startReplace; i++) {
- spaces.append(' ');
+ private static void replaceCommonSpecialCharacters(StringBuilder source) {
+
+ for (int i = 0; i < SPEC_PATTERN_REPLACEMENTS.length; i++) {
+ Pattern pattern = SPEC_PATTERN_REPLACEMENTS[i];
+ Matcher m = pattern.matcher(source);
+ while (m.find()) {
+ for (int idx = m.start(); idx < m.end(); idx++) {
+ source.setCharAt(idx, '_');
}
- result.replace(startReplace, endReplace + 1,
spaces.toString());
- startReplace = result.indexOf("{{", endReplace);
- } else {
- startReplace = -1;
}
}
- return result.toString();
- }
-
- private static String replaceCommonSpecialCharacters(String source) {
- source = source.replace('@', '_'); //NOI18N
- source = source.replace('?', '_'); //NOI18N
- source = source.replaceAll("!(?!(omap|!omap))", "_"); //NOI18N
- return source;
}
private static String replaceInlineRegexBrackets(String source) {
@@ -180,43 +165,55 @@ public class YamlParser extends Parser {
}
// for test package private
+ YamlParserResult parse(String src, Snapshot snapshot) {
- YamlParserResult parse(String source, Snapshot snapshot) {
+ if (isTooLarge(src)) {
+ return resultForTooLargeFile(snapshot);
+ }
- source = replacePhpFragments(source);
- source = replaceMustache(source);
- source = replaceCommonSpecialCharacters(source);
- source = replaceInlineRegexBrackets(source);
+ StringBuilder sb = new StringBuilder(src);
+ replacePhpFragments(sb);
+ replaceMustache(sb);
+ replaceCommonSpecialCharacters(sb);
+// source = replaceInlineRegexBrackets(source);
- List<Node> nodes = new ArrayList<Node>();
- try {
- if (isTooLarge(source)) {
- return resultForTooLargeFile(snapshot);
- }
- LoadSettings settings = LoadSettings.builder().build();
- ScannerImpl scanner = new ScannerImpl(settings, new
StreamReader(settings, source));
- ParserImpl parser = new ParserImpl(settings, scanner);
- Composer composer = new Composer(settings, parser);
+ YamlParserResult result = new YamlParserResult(snapshot, false);
+
+ Deque<YamlSection> sources = new LinkedList<>();
+ sources.push(new YamlSection(sb.toString()));
+ while (!sources.isEmpty()) {
- while (composer.hasNext()) {
- Node node = composer.next();
- if (node == null) {
- break;
+ YamlSection section = sources.pop();
+ try {
+ List<? extends StructureItem> items = section.collectItems();
+ result.addStructure(items);
+ } catch (ScannerException se) {
+ result.addError(section.processScannerException(snapshot, se));
+ YamlSection after =
section.after(se.getProblemMark().get().getIndex());
+ YamlSection before =
section.before(se.getContextMark().get().getIndex());
+ if (!after.isEmpty()) sources.push(after);
+ if (!before.isEmpty()) sources.push(before);
+ if ((before.getLength() + after.getLength()) ==
section.getLength()) {
+ LOGGER.info("Chanche to loop forever");
+ }
+ } catch (ParserException pe ){
+ result.addError(section.processParserException(snapshot, pe));
+ YamlSection after =
section.after(pe.getProblemMark().get().getIndex());
+ YamlSection before =
section.before(pe.getProblemMark().get().getIndex());
+ if (!after.isEmpty()) sources.push(after);
+ if (!before.isEmpty()) sources.push(before);
+ if ((before.getLength() + after.getLength()) ==
section.getLength()) {
+ LOGGER.info("Chanche to loop forever");
+ }
+ } catch (Exception ex) {
+ String message = ex.getMessage();
+ if (message != null && message.length() > 0) {
+ result.addError(processError(message, snapshot, 0));
}
- nodes.add(node);
- }
- //TODO: add errors
- YamlParserResult result = new YamlParserResult(nodes, this,
snapshot, true);
- return result;
- } catch (Exception ex) {
- YamlParserResult result = new
YamlParserResult(Collections.<Node>emptyList(), this, snapshot, false);
- String message = ex.getMessage();
- if (message != null && message.length() > 0) {
- result.addError(processError(message, snapshot, 0));
}
- return result;
}
- }
+ return result;
+ }
private DefaultError processError(String message, Snapshot snapshot, int
pos) {
// Strip off useless prefixes to make errors more readable
@@ -294,7 +291,7 @@ public class YamlParser extends Parser {
lastResult = parse(source, snapshot);
} catch (Exception ioe) {
- lastResult = new YamlParserResult(Collections.<Node>emptyList(),
this, snapshot, false);
+ lastResult = new YamlParserResult(snapshot, false);
}
}
}
diff --git
a/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlParserResult.java
b/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlParserResult.java
index e289ecb..14a383a 100644
---
a/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlParserResult.java
+++
b/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlParserResult.java
@@ -22,11 +22,9 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.netbeans.modules.csl.api.Error;
-import org.netbeans.modules.csl.api.OffsetRange;
import org.netbeans.modules.csl.api.StructureItem;
import org.netbeans.modules.csl.spi.ParserResult;
import org.netbeans.modules.parsing.api.Snapshot;
-import org.snakeyaml.engine.v2.nodes.Node;
/**
* A result from Parsing YAML
@@ -35,24 +33,21 @@ import org.snakeyaml.engine.v2.nodes.Node;
*/
public class YamlParserResult extends ParserResult {
- private final List<Error> errors = new ArrayList<Error>();
- private List<Node> nodes;
- private List<? extends StructureItem> items;
+ private final List<Error> errors = new ArrayList<>();
+ private final List<StructureItem> structure = new ArrayList<>();
- public YamlParserResult(List<Node> nodes, YamlParser parser, Snapshot
snapshot, boolean valid) {
+ public YamlParserResult(Snapshot snapshot, boolean valid) {
super(snapshot);
- assert nodes != null;
- this.nodes = nodes;
- }
-
- public List<Node> getRootNodes() {
- return nodes;
}
public void addError(Error error) {
errors.add(error);
}
+ public void addStructure(List<? extends StructureItem> items) {
+ structure.addAll(items);
+ }
+
@Override
public List<? extends Error> getDiagnostics() {
return Collections.unmodifiableList(errors);
@@ -65,20 +60,7 @@ public class YamlParserResult extends ParserResult {
}
public synchronized List<? extends StructureItem> getItems() {
- if (items == null) {
- items = new YamlScanner().scanStructure(this);
- }
-
- return items;
- }
-
- public void setItems(List<? extends StructureItem> items) {
- this.items = items;
+ return structure;
}
- public static OffsetRange getAstRange(Node node) {
- int s = node.getStartMark().get().getPointer();
- int e = node.getEndMark().get().getPointer();
- return new OffsetRange(s, e);
- }
}
diff --git
a/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlScanner.java
b/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlScanner.java
index 495a4fc..8237d83 100644
---
a/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlScanner.java
+++
b/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlScanner.java
@@ -18,24 +18,15 @@
*/
package org.netbeans.modules.languages.yaml;
-import java.io.CharConversionException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-import java.util.logging.Level;
import java.util.logging.Logger;
-import javax.swing.ImageIcon;
import javax.swing.text.BadLocationException;
import org.netbeans.editor.BaseDocument;
import org.netbeans.editor.Utilities;
-import org.netbeans.modules.csl.api.ElementHandle;
-import org.netbeans.modules.csl.api.ElementKind;
-import org.netbeans.modules.csl.api.HtmlFormatter;
-import org.netbeans.modules.csl.api.Modifier;
import org.netbeans.modules.csl.api.OffsetRange;
import org.netbeans.modules.csl.api.StructureItem;
import org.netbeans.modules.csl.api.StructureScanner;
@@ -43,8 +34,6 @@ import
org.netbeans.modules.csl.api.StructureScanner.Configuration;
import org.netbeans.modules.csl.spi.GsfUtilities;
import org.netbeans.modules.csl.spi.ParserResult;
import org.openide.util.Exceptions;
-import org.openide.xml.XMLUtil;
-import org.snakeyaml.engine.v2.nodes.Node;
/**
* Structure Scanner for YAML
@@ -58,54 +47,34 @@ public class YamlScanner implements StructureScanner {
@Override
public List<? extends StructureItem> scan(ParserResult info) {
YamlParserResult result = (YamlParserResult) info;
- if (result != null) {
- return result.getItems();
- }
-
- return Collections.emptyList();
- }
-
- List<? extends StructureItem> scanStructure(YamlParserResult result) {
- List<Node> nodes = result.getRootNodes();
- if (nodes.size() > 0) {
- // Skip root node
- return YamlStructureItem.initialize(result, nodes);
- }
-
- return Collections.emptyList();
+ return result.getItems();
}
@Override
public Map<String, List<OffsetRange>> folds(ParserResult info) {
- YamlParserResult result = (YamlParserResult) info;
- if (result == null) {
- return Collections.emptyMap();
- }
-
- List<? extends StructureItem> items = result.getItems();
- if (items.isEmpty()) {
- return Collections.emptyMap();
- }
-
- Map<String, List<OffsetRange>> folds = new HashMap<>();
- List<OffsetRange> codeblocks = new ArrayList<>();
- folds.put("tags", codeblocks); // NOI18N
-
- BaseDocument doc = (BaseDocument)
result.getSnapshot().getSource().getDocument(false);
-
- for (StructureItem item : items) {
- try {
- addBlocks(result, doc, result.getSnapshot().getText(),
codeblocks, item);
- } catch (BadLocationException ble) {
- Exceptions.printStackTrace(ble);
- break;
+ Map<String, List<OffsetRange>> folds = Collections.emptyMap();
+ List<? extends StructureItem> items = scan(info);
+
+ if (!items.isEmpty()) {
+ folds = new HashMap<>();
+ List<OffsetRange> codeblocks = new ArrayList<>();
+ folds.put("tags", codeblocks); // NOI18N
+
+ BaseDocument doc = (BaseDocument)
info.getSnapshot().getSource().getDocument(false);
+
+ for (StructureItem item : items) {
+ try {
+ addBlocks(doc, info.getSnapshot().getText(), codeblocks,
item);
+ } catch (BadLocationException ble) {
+ Exceptions.printStackTrace(ble);
+ break;
+ }
}
}
-
return folds;
}
- private void addBlocks(YamlParserResult result, BaseDocument doc,
CharSequence text, List<OffsetRange> codeblocks, StructureItem item) throws
BadLocationException {
+ private void addBlocks(BaseDocument doc, CharSequence text,
List<OffsetRange> codeblocks, StructureItem item) throws BadLocationException {
int docLength = doc == null ? text.length() : doc.getLength();
int begin = Math.min((int) item.getPosition(), docLength);
int end = Math.min((int) item.getEndPosition(), docLength);
@@ -121,7 +90,7 @@ public class YamlScanner implements StructureScanner {
int childBegin = (int) child.getPosition();
int childEnd = (int) child.getEndPosition();
if (childBegin >= begin && childEnd <= end) {
- addBlocks(result, doc, text, codeblocks, child);
+ addBlocks(doc, text, codeblocks, child);
}
}
}
@@ -131,116 +100,4 @@ public class YamlScanner implements StructureScanner {
return new Configuration(false, false, 0);
}
- private static class YamlStructureItem implements StructureItem,
Comparable<YamlStructureItem> {
-
- private final String name;
- private List<YamlStructureItem> children;
- private final Node node;
-
- YamlStructureItem(Node node, String name) {
- this.node = node;
- this.name = name;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public String getSortText() {
- return getName();
- }
-
- @Override
- public String getHtml(HtmlFormatter formatter) {
- String s = getName();
- try {
- return XMLUtil.toElementContent(s);
- } catch (CharConversionException cce) {
- // fine to just log...probably some UTF8 name (e.g. russian
cyrillic, etc.)
- LOGGER.log(Level.FINE, "NAME:" + s, cce);
- return s;
- }
- }
-
- @Override
- public ElementHandle getElementHandle() {
- return null;
- }
-
- @Override
- public ElementKind getKind() {
- return ElementKind.ATTRIBUTE;
- }
-
- @Override
- public Set<Modifier> getModifiers() {
- return Collections.emptySet();
- }
-
- @Override
- public boolean isLeaf() {
- return getNestedItems().isEmpty();
- }
-
- private static List<? extends StructureItem>
initialize(YamlParserResult result, List<Node> roots) {
- // Really need IdentitySet or IdentityHashSet but there isn't one
built in
- // or in our available libraries...
- Set<Node> seen = new HashSet<>(100);
- List<StructureItem> children = new ArrayList<>();
- for (Node root : roots) {
- YamlStructureItem fakeRoot = new YamlStructureItem(root, null);
- initializeChildren(result, fakeRoot, seen, 0);
- children.addAll(fakeRoot.children);
- }
- return children;
- }
-
- @SuppressWarnings("unchecked")
- private static void initializeChildren(YamlParserResult result,
YamlStructureItem item, Set<Node> seen, int depth) {
- Node node = item.node;
- if (seen.contains(node)) {
- return;
- }
- seen.add(node);
-
- switch (node.getNodeType()) {
- case MAPPING:
- break;
- case SEQUENCE:
- break;
- case SCALAR:
- break;
- }
- item.children = Collections.emptyList();
- }
-
- @Override
- public List<? extends StructureItem> getNestedItems() {
- assert children != null;
- return children;
- }
-
- @Override
- public long getPosition() {
- return node.getStartMark().get().getPointer();
- }
-
- @Override
- public long getEndPosition() {
- return node.getEndMark().get().getPointer();
- }
-
- @Override
- public ImageIcon getCustomIcon() {
- return null;
- }
-
- @Override
- public int compareTo(YamlStructureItem other) {
- return (int) (getPosition() - other.getPosition());
- }
-
- }
}
diff --git
a/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlSection.java
b/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlSection.java
new file mode 100644
index 0000000..0c73f44
--- /dev/null
+++
b/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlSection.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.netbeans.modules.languages.yaml;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import org.netbeans.modules.csl.api.Severity;
+import org.netbeans.modules.csl.api.StructureItem;
+import org.netbeans.modules.csl.spi.DefaultError;
+import org.netbeans.modules.parsing.api.Snapshot;
+import org.snakeyaml.engine.v2.api.LoadSettings;
+import org.snakeyaml.engine.v2.events.AliasEvent;
+import org.snakeyaml.engine.v2.events.Event;
+import org.snakeyaml.engine.v2.events.MappingEndEvent;
+import org.snakeyaml.engine.v2.events.MappingStartEvent;
+import org.snakeyaml.engine.v2.events.ScalarEvent;
+import org.snakeyaml.engine.v2.events.SequenceEndEvent;
+import org.snakeyaml.engine.v2.events.SequenceStartEvent;
+import org.snakeyaml.engine.v2.exceptions.Mark;
+import org.snakeyaml.engine.v2.exceptions.ParserException;
+import org.snakeyaml.engine.v2.exceptions.ScannerException;
+import org.snakeyaml.engine.v2.parser.Parser;
+import org.snakeyaml.engine.v2.parser.ParserImpl;
+import org.snakeyaml.engine.v2.scanner.ScannerImpl;
+import org.snakeyaml.engine.v2.scanner.StreamReader;
+
+import static org.netbeans.modules.languages.yaml.YamlStructureItem.NodeType.*;
+/**
+ *
+ * @author lkishalmi
+ */
+public class YamlSection {
+
+ private static final LoadSettings SETTINGS =
LoadSettings.builder().build();
+ final int offset;
+ final String source;
+
+ private Parser parser = null;
+
+ YamlSection(int offset, String source) {
+ this.offset = offset;
+ this.source = source;
+ }
+
+ public YamlSection(String source) {
+ this(0, source);
+ }
+
+ public YamlSection before(int index) {
+ return new YamlSection(offset, source.substring(0, index));
+ }
+
+ public YamlSection after(int index) {
+ return new YamlSection(offset + index, source.substring(index));
+ }
+
+ public boolean isEmpty() {
+ return source.isEmpty();
+ }
+
+ public int getLength() {
+ return source.length();
+ }
+
+ List<? extends StructureItem> collectItems() {
+ if (parser != null) {
+ throw new IllegalStateException("This YAML segment is already
parsed.");
+ }
+ List< StructureItem> ret = new ArrayList<>();
+ ScannerImpl scanner = new ScannerImpl(SETTINGS, new
StreamReader(SETTINGS, source));
+ parser = new ParserImpl(SETTINGS, scanner);
+ while (parser.hasNext()) {
+ YamlStructureItem root = processItem();
+ if (root != null) {
+ ret.addAll(root.getNestedItems());
+ }
+ }
+ return ret;
+ }
+
+ private YamlStructureItem processItem() {
+ YamlStructureItem ret = null;
+ while ((ret == null) && parser.hasNext()) {
+ switch (parser.peekEvent().getEventId()) {
+ case MappingStart:
+ ret = processMapping((MappingStartEvent) parser.next());
+ break;
+ case SequenceStart:
+ ret = processSequence((SequenceStartEvent) parser.next());
+ break;
+ case Scalar:
+ ret = processScalar((ScalarEvent) parser.next());
+ break;
+ case Alias:
+ ret = processAlias((AliasEvent) parser.next());
+ break;
+ default:
+ parser.next();
+ }
+ }
+ return ret;
+ }
+
+ private YamlStructureItem processAlias(AliasEvent evt) {
+ return new YamlStructureItem.Simple(ALIAS, evt.getAlias().getValue(),
getIndex(evt.getStartMark()), getIndex(evt.getEndMark()));
+ }
+
+ private YamlStructureItem processScalar(ScalarEvent evt) {
+ return new YamlStructureItem.Simple(SCALAR,evt.getValue(),
getIndex(evt.getStartMark()), getIndex(evt.getEndMark()));
+ }
+
+
+ private YamlStructureItem processMapping(MappingStartEvent evt) {
+ YamlStructureItem.Collection item = new
YamlStructureItem.Collection(MAP, getIndex(evt.getStartMark()));
+ while (parser.hasNext() && !parser.checkEvent(Event.ID.MappingEnd)) {
+ YamlStructureItem keyItem = processItem();
+ YamlStructureItem valueItem = processItem();
+ item.add(new YamlStructureItem.MapEntry(keyItem, valueItem));
+ }
+ if (parser.hasNext()) {
+ MappingEndEvent eevt = (MappingEndEvent) parser.next();
+ if (evt.isFlow()) {
+ item.setEndMark(getIndex(eevt.getEndMark()));
+ }
+ }
+ return item;
+ }
+
+ private YamlStructureItem processSequence(SequenceStartEvent evt) {
+ YamlStructureItem.Collection item = new
YamlStructureItem.Collection(SEQUENCE, getIndex(evt.getStartMark()));
+ while (parser.hasNext() && !parser.checkEvent(Event.ID.SequenceEnd)) {
+ item.add(processItem());
+ }
+ if (parser.hasNext()) {
+ SequenceEndEvent eevt = (SequenceEndEvent) parser.next();
+ if (evt.isFlow()) {
+ item.setEndMark(getIndex(eevt.getEndMark()));
+ }
+ }
+ return item;
+ }
+
+ DefaultError processScannerException(Snapshot snapshot, ScannerException
se) {
+ int contextIndex = getIndex(se.getContextMark());
+ int problemIndex = getIndex(se.getProblemMark());
+ StringBuilder message = new StringBuilder(se.getContext());
+ message.append(", ").append(se.getProblem());
+ char upper = Character.toUpperCase(message.charAt(0));
+ message.setCharAt(0, upper);
+ return new DefaultError(null, message.toString(), null,
snapshot.getSource().getFileObject(), contextIndex, problemIndex,
Severity.ERROR);
+ }
+
+ DefaultError processParserException(Snapshot snapshot, ParserException se)
{
+ int problemIndex = getIndex(se.getProblemMark());
+ StringBuilder message = new StringBuilder(se.getContext());
+ message.append(", ").append(se.getProblem());
+ char upper = Character.toUpperCase(message.charAt(0));
+ message.setCharAt(0, upper);
+ return new DefaultError(null, message.toString(), null,
snapshot.getSource().getFileObject(), problemIndex, problemIndex,
Severity.ERROR);
+ }
+
+ private int getIndex(Optional<Mark> om) {
+ return om.get().getIndex() + offset;
+ }
+
+ @Override
+ public String toString() {
+ return "" + offset + ":" + source;
+ }
+}
diff --git
a/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlSemanticAnalyzer.java
b/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlSemanticAnalyzer.java
index 35db1e9..69af95c 100644
---
a/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlSemanticAnalyzer.java
+++
b/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlSemanticAnalyzer.java
@@ -20,23 +20,15 @@ package org.netbeans.modules.languages.yaml;
import java.util.Collections;
import java.util.HashMap;
-import java.util.IdentityHashMap;
-import java.util.List;
import java.util.Map;
import java.util.Set;
import org.netbeans.modules.csl.api.ColoringAttributes;
import org.netbeans.modules.csl.api.OffsetRange;
import org.netbeans.modules.csl.api.SemanticAnalyzer;
+import org.netbeans.modules.csl.api.StructureItem;
import org.netbeans.modules.parsing.spi.Parser.Result;
import org.netbeans.modules.parsing.spi.Scheduler;
import org.netbeans.modules.parsing.spi.SchedulerEvent;
-import org.snakeyaml.engine.v2.nodes.MappingNode;
-import org.snakeyaml.engine.v2.nodes.Node;
-import org.snakeyaml.engine.v2.nodes.NodeTuple;
-
-import static org.snakeyaml.engine.v2.nodes.NodeType.*;
-import org.snakeyaml.engine.v2.nodes.ScalarNode;
-import org.snakeyaml.engine.v2.nodes.SequenceNode;
/**
* Semantic Analyzer for YAML
@@ -85,54 +77,47 @@ public class YamlSemanticAnalyzer extends SemanticAnalyzer {
}
YamlParserResult ypr = (YamlParserResult) result;
- if (ypr == null || ypr.getRootNodes().isEmpty()) {
+ if (ypr == null || ypr.getItems().isEmpty()) {
this.semanticHighlights = Collections.emptyMap();
return;
}
- List<Node> rootNodes = ypr.getRootNodes();
-
Map<OffsetRange, Set<ColoringAttributes>> highlights =
new HashMap<OffsetRange, Set<ColoringAttributes>>(100);
- IdentityHashMap<Object, Boolean> seen = new IdentityHashMap<Object,
Boolean>(100);
- for (Node root : rootNodes) {
- addHighlights(ypr, root, highlights, seen, 0);
+ for (StructureItem item : ypr.getItems()) {
+ YamlStructureItem yamlItem = (YamlStructureItem) item;
+ addHighlights(yamlItem, highlights);
}
this.semanticHighlights = highlights;
}
- private void addHighlights(YamlParserResult ypr, Node node,
Map<OffsetRange, Set<ColoringAttributes>> highlights, IdentityHashMap<Object,
Boolean> seen, int depth) {
- if (depth > 10 || node == null) {
- // Avoid boundless recursion; some datastructures from YAML appear
to be recursive
- return;
- }
- if (seen.containsKey(node)) {
- return;
- }
- seen.put(node, Boolean.TRUE);
-
- switch (node.getNodeType()) {
- case MAPPING:
- MappingNode mappings = (MappingNode) node;
- List<NodeTuple> tuples = mappings.getValue();
- for (NodeTuple tuple : tuples) {
- addHighlights(ypr, tuple.getValueNode(), highlights, seen,
depth + 1);
- }
- break;
+ private void addHighlights(YamlStructureItem item, Map<OffsetRange,
Set<ColoringAttributes>> highlights) {
+ switch (item.getType()) {
+ case MAP:
case SEQUENCE:
- SequenceNode sequence = (SequenceNode) node;
- List<Node> nodes = sequence.getValue();
- for (Node node1 : nodes) {
- addHighlights(ypr, node, highlights, seen, depth + 1);
+ YamlStructureItem.Collection coll =
(YamlStructureItem.Collection) item;
+ for (YamlStructureItem child : coll.getChildren()) {
+ addHighlights(child, highlights);
}
break;
- case SCALAR:
- ScalarNode scalar = (ScalarNode) node;
- OffsetRange range = YamlParserResult.getAstRange(scalar);
- highlights.put(range, ColoringAttributes.METHOD_SET);
+ case MAPPING:
+ YamlStructureItem.MapEntry entry =
(YamlStructureItem.MapEntry) item;
+ if (entry.keyItem.getType() ==
YamlStructureItem.NodeType.SCALAR) {
+ highlights.put(getAstRange(entry.keyItem),
ColoringAttributes.METHOD_SET);
+ } else {
+ addHighlights(entry.keyItem, highlights);
+ }
+ addHighlights(entry.valueItem, highlights);
break;
}
}
+
+ private static OffsetRange getAstRange(YamlStructureItem item) {
+ int s = (int) item.getPosition();
+ int e = (int) item.getEndPosition();
+ return new OffsetRange(s, e);
+ }
+
}
diff --git
a/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlStructureItem.java
b/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlStructureItem.java
new file mode 100644
index 0000000..4412d43
--- /dev/null
+++
b/ide/languages.yaml/src/org/netbeans/modules/languages/yaml/YamlStructureItem.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.netbeans.modules.languages.yaml;
+
+import java.io.CharConversionException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.swing.ImageIcon;
+import org.netbeans.modules.csl.api.ElementHandle;
+import org.netbeans.modules.csl.api.ElementKind;
+import org.netbeans.modules.csl.api.HtmlFormatter;
+import org.netbeans.modules.csl.api.Modifier;
+import org.netbeans.modules.csl.api.StructureItem;
+import org.openide.xml.XMLUtil;
+
+/**
+ *
+ * @author lkishalmi
+ */
+public abstract class YamlStructureItem implements StructureItem {
+
+ public static enum NodeType { MAP, SEQUENCE, MAPPING, SCALAR, ALIAS };
+
+ private static final Logger LOGGER =
Logger.getLogger(YamlStructureItem.class.getName());
+ private final NodeType type;
+ private final long startMark;
+
+ public YamlStructureItem(NodeType type, long startMark) {
+ this.type = type;
+ this.startMark = startMark;
+ }
+
+ public NodeType getType() {
+ return type;
+ }
+
+ @Override
+ public String getSortText() {
+ return getName();
+ }
+
+ @Override
+ public String getHtml(HtmlFormatter formatter) {
+ try {
+ return XMLUtil.toElementContent(getName());
+ } catch (CharConversionException cce) {
+ // fine to just log...probably some UTF8 name (e.g. russian
cyrillic, etc.)
+ LOGGER.log(Level.FINE, "NAME:" + getName(), cce);
+ return getName();
+ }
+ }
+
+ @Override
+ public ElementHandle getElementHandle() {
+ return null;
+ }
+
+ @Override
+ public ElementKind getKind() {
+ return ElementKind.ATTRIBUTE;
+ }
+
+ @Override
+ public Set<Modifier> getModifiers() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public boolean isLeaf() {
+ return getNestedItems().isEmpty();
+ }
+
+ @Override
+ public long getPosition() {
+ return startMark;
+ }
+
+ @Override
+ public ImageIcon getCustomIcon() {
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return "" + getPosition() + ": " + getName();
+ }
+
+ public final static class Simple extends YamlStructureItem {
+
+ final String name;
+ final long endMark;
+
+ public Simple(NodeType type, String name, long startMark, long
endMark) {
+ super(type, startMark);
+ this.name = name;
+ this.endMark = endMark;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public List<? extends StructureItem> getNestedItems() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public long getEndPosition() {
+ return endMark;
+ }
+
+ }
+
+ public final static class Collection extends YamlStructureItem {
+ private final List<YamlStructureItem> children = new ArrayList<>();
+ private long endMark;
+
+ public Collection(NodeType type, long startMark) {
+ super(type, startMark);
+ this.endMark = startMark;
+ }
+
+ @Override
+ public String getName() {
+ return "list item";
+ }
+
+ @Override
+ public List<? extends StructureItem> getNestedItems() {
+ return children;
+ }
+
+ public List<? extends YamlStructureItem> getChildren() {
+ return children;
+ }
+
+ @Override
+ public long getEndPosition() {
+ return endMark;
+ }
+
+ public void add(YamlStructureItem item) {
+ endMark = item.getEndPosition();
+ children.add(item);
+ }
+
+ public void setEndMark(int end) {
+ endMark = end;
+ }
+ }
+
+ public final static class MapEntry extends YamlStructureItem {
+ final YamlStructureItem keyItem;
+ final YamlStructureItem valueItem;
+
+ public MapEntry(YamlStructureItem keyItem, YamlStructureItem
valueItem) {
+ super(NodeType.MAPPING, keyItem.startMark);
+ this.keyItem = keyItem;
+ this.valueItem = valueItem;
+ }
+
+ @Override
+ public String getName() {
+ return keyItem.getName();
+ }
+
+ @Override
+ public List<? extends StructureItem> getNestedItems() {
+ return valueItem.getNestedItems();
+ }
+
+ @Override
+ public long getEndPosition() {
+ return valueItem.getEndPosition();
+ }
+ }
+
+}
diff --git a/ide/languages.yaml/test/unit/data/testfiles/error.yaml.errors
b/ide/languages.yaml/test/unit/data/testfiles/error.yaml.errors
index 56a5dde..f1d0ebf 100644
--- a/ide/languages.yaml/test/unit/data/testfiles/error.yaml.errors
+++ b/ide/languages.yaml/test/unit/data/testfiles/error.yaml.errors
@@ -1 +1 @@
-152-152:While scanning a block mapping we had this expected <block end>, but
found org.jvyamlb.tokens.PositionedScalarToken
+152-159:While scanning a simple key, could not find expected ':'
diff --git a/ide/languages.yaml/test/unit/data/testfiles/error2.yaml.errors
b/ide/languages.yaml/test/unit/data/testfiles/error2.yaml.errors
index 5603088..ca350bc 100644
--- a/ide/languages.yaml/test/unit/data/testfiles/error2.yaml.errors
+++ b/ide/languages.yaml/test/unit/data/testfiles/error2.yaml.errors
@@ -1 +1 @@
-272-272:While scanning a block mapping we had this expected <block end>, but
found org.jvyamlb.tokens.PositionedScalarToken
+272-279:While scanning a simple key, could not find expected ':'
diff --git a/ide/languages.yaml/test/unit/data/testfiles/error3.yaml.errors
b/ide/languages.yaml/test/unit/data/testfiles/error3.yaml.errors
index 38da867..ebfa576 100644
--- a/ide/languages.yaml/test/unit/data/testfiles/error3.yaml.errors
+++ b/ide/languages.yaml/test/unit/data/testfiles/error3.yaml.errors
@@ -1 +1 @@
-31-31:While scanning a block mapping we had this expected <block end>, but
found org.jvyamlb.tokens.PositionedBlockMappingStartToken
+31-31:While parsing a block mapping, expected <block end>, but found '<block
mapping start>'
diff --git a/ide/languages.yaml/test/unit/data/testfiles/fixture3.yml.structure
b/ide/languages.yaml/test/unit/data/testfiles/fixture3.yml.structure
index 0b359ee..c3cfeb5 100644
--- a/ide/languages.yaml/test/unit/data/testfiles/fixture3.yml.structure
+++ b/ide/languages.yaml/test/unit/data/testfiles/fixture3.yml.structure
@@ -2,9 +2,7 @@ DEFAULTS:ATTRIBUTE:[]:DEFAULTS:
created_on:ATTRIBUTE:[]:created_on:
first:ATTRIBUTE:[]:first:
<<:ATTRIBUTE:[]:<<:
- created_on:ATTRIBUTE:[]:created_on:
name:ATTRIBUTE:[]:name:
second:ATTRIBUTE:[]:second:
<<:ATTRIBUTE:[]:<<:
- created_on:ATTRIBUTE:[]:created_on:
name:ATTRIBUTE:[]:name:
diff --git a/ide/languages.yaml/test/unit/data/testfiles/test5.yaml.structure
b/ide/languages.yaml/test/unit/data/testfiles/test5.yaml.structure
index cfd7817..5a30443 100644
--- a/ide/languages.yaml/test/unit/data/testfiles/test5.yaml.structure
+++ b/ide/languages.yaml/test/unit/data/testfiles/test5.yaml.structure
@@ -11,17 +11,9 @@ children:ATTRIBUTE:[]:children:
age:ATTRIBUTE:[]:age:
name:ATTRIBUTE:[]:name:
siblings:ATTRIBUTE:[]:siblings:
- list item:ATTRIBUTE:[]:list item:
- age:ATTRIBUTE:[]:age:
- children:ATTRIBUTE:[]:children:
- name:ATTRIBUTE:[]:name:
- siblings:ATTRIBUTE:[]:siblings:
- spouse:ATTRIBUTE:[]:spouse:
+ 1:ATTRIBUTE:[]:1:
list item:ATTRIBUTE:[]:list item:
age:ATTRIBUTE:[]:age:
name:ATTRIBUTE:[]:name:
siblings:ATTRIBUTE:[]:siblings:
- list item:ATTRIBUTE:[]:list item:
- age:ATTRIBUTE:[]:age:
- name:ATTRIBUTE:[]:name:
- siblings:ATTRIBUTE:[]:siblings:
+ 2:ATTRIBUTE:[]:2:
\ No newline at end of file
diff --git
a/ide/languages.yaml/test/unit/src/org/netbeans/modules/languages/yaml/YamlParserTest.java
b/ide/languages.yaml/test/unit/src/org/netbeans/modules/languages/yaml/YamlParserTest.java
index e4d8663..e17982d 100644
---
a/ide/languages.yaml/test/unit/src/org/netbeans/modules/languages/yaml/YamlParserTest.java
+++
b/ide/languages.yaml/test/unit/src/org/netbeans/modules/languages/yaml/YamlParserTest.java
@@ -87,7 +87,7 @@ public class YamlParserTest extends YamlTestBase {
String annotatedSource = annotateErrors(diagnostics);
assertDescriptionMatches("testfiles/" + relFilePath,
annotatedSource, false, ".errors", false);
// Make sure we actually skipped parsing this large document!
- assertTrue(result.getRootNodes().size() == 0);
+ assertTrue(result.getItems().isEmpty());
}
});
}
@@ -110,21 +110,27 @@ public class YamlParserTest extends YamlTestBase {
assertNotNull("Parser result must be nonnull", parser.getResult(null));
}
+ private String replacePhpFragments(String source) {
+ StringBuilder sb = new StringBuilder(source);
+ YamlParser.replacePhpFragments(sb);
+ return sb.toString();
+ }
+
public void testReplacePhpFragments() {
- assertEquals("", YamlParser.replacePhpFragments(""));
- assertEquals("foo bar", YamlParser.replacePhpFragments("foo bar"));
- assertEquals("?>", YamlParser.replacePhpFragments("?>"));
- assertEquals("<?", YamlParser.replacePhpFragments("<?"));
- assertEquals("foo ?>", YamlParser.replacePhpFragments("foo ?>"));
- assertEquals("<? bar", YamlParser.replacePhpFragments("<? bar"));
-
- assertEquals(" ", YamlParser.replacePhpFragments("<??>"));
- assertEquals("foo: ", YamlParser.replacePhpFragments("foo:<??>"));
- assertEquals("foo: ",
YamlParser.replacePhpFragments("foo:<? here goes php ?>"));
- assertEquals("foo baz", YamlParser.replacePhpFragments("foo
<? bar ?> baz"));
-
- assertEquals(" ", YamlParser.replacePhpFragments("<??><??>"));
- assertEquals("foo: bar: qux",
YamlParser.replacePhpFragments("foo:<??>bar:<?baz?>qux"));
+ assertEquals("", replacePhpFragments(""));
+ assertEquals("foo bar", replacePhpFragments("foo bar"));
+ assertEquals("?>", replacePhpFragments("?>"));
+ assertEquals("<?", replacePhpFragments("<?"));
+ assertEquals("foo ?>", replacePhpFragments("foo ?>"));
+ assertEquals("<? bar", replacePhpFragments("<? bar"));
+
+ assertEquals(" ", replacePhpFragments("<??>"));
+ assertEquals("foo: ", replacePhpFragments("foo:<??>"));
+ assertEquals("foo: ", replacePhpFragments("foo:<?
here goes php ?>"));
+ assertEquals("foo baz", replacePhpFragments("foo <? bar ?>
baz"));
+
+ assertEquals(" ", replacePhpFragments("<??><??>"));
+ assertEquals("foo: bar: qux",
replacePhpFragments("foo:<??>bar:<?baz?>qux"));
}
public void testReplacePhpFragmentsPerformance() {
@@ -136,11 +142,11 @@ public class YamlParserTest extends YamlTestBase {
source.append("<? php here ?>");
}
}
- long start = System.currentTimeMillis();
- YamlParser.replacePhpFragments(source.toString());
- long time = System.currentTimeMillis() - start;
+ long start = System.nanoTime();
+ YamlParser.replacePhpFragments(source);
+ long time = System.nanoTime() - start;
// takes about 30 ms on my laptop, so I suppose 300 ms should
// be enough pretty much on any machine
- assertTrue("Slow replacing of php fragments: " + time + " ms", time <
300);
+ assertTrue("Slow replacing of php fragments: " + time + " ms", time <
300_000_000L);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@netbeans.apache.org
For additional commands, e-mail: commits-h...@netbeans.apache.org
For further information about the NetBeans mailing lists, visit:
https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists
