Taewoo Kim has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1431
Change subject: Add some functionalities to Grammar Extension plugin
......................................................................
Add some functionalities to Grammar Extension plugin
- Can add a Java method at the end of the class definition
- Extend @merge replace "oldphrase" with "newphrase"
- Can add a construct at the end of the file
- Can deal with class modifier
Change-Id: Icafc01ebe591b81b3c3f69ea9da123dbfad19bb0
---
M
asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
M
asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
M
asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
3 files changed, 219 insertions(+), 25 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/31/1431/1
diff --git
a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
index 5b44e23..a77374e 100644
---
a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
+++
b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
@@ -68,16 +68,23 @@
private static final String KWUNIMPORT = "unimport";
private static final String KWPACKAGE = "package";
private static final String NEWPRODUCTION = "@new";
+ // Adds a construct at the end of the file.
+ private static final String NEW_AT_THE_END_PRODUCTION = "@new_at_the_end";
+ // Adds a method at the end of the class definition.
+ private static final String NEW_AT_THE_END_CLASS_DEFINITION =
"@new_at_the_class_def";
private static final String MERGEPRODUCTION = "@merge";
private static final String OVERRIDEPRODUCTION = "@override";
private static final String BEFORE = "before:";
private static final String AFTER = "after:";
private static final String REPLACE = "replace";
private static final String WITH = "with";
+ private static final String OPTION_TRUE = "true";
+ private static final String OPTION_FALSE = "false";
private static final List<String> KEYWORDS = Arrays
.asList(new String[] { KWCLASS, KWIMPORT, KWPACKAGE, PARSER_BEGIN,
PARSER_END });
- private static final List<String> EXTENSIONKEYWORDS = Arrays
- .asList(new String[] { KWIMPORT, KWUNIMPORT, NEWPRODUCTION,
OVERRIDEPRODUCTION, MERGEPRODUCTION });
+ private static final List<String> EXTENSIONKEYWORDS =
+ Arrays.asList(new String[] { KWIMPORT, KWUNIMPORT, NEWPRODUCTION,
NEW_AT_THE_END_PRODUCTION,
+ NEW_AT_THE_END_CLASS_DEFINITION, OVERRIDEPRODUCTION,
MERGEPRODUCTION });
private static final String REGEX_WS_DOT_SEMICOLON = "\\s|[.]|[;]";
private static final String REGEX_WS_PAREN = "\\s|[(]|[)]";
private static final String OPTIONS = "options";
@@ -87,14 +94,20 @@
private Map<String, String[]> mergeElements = new HashMap<>();
private List<Pair<String, String>> baseFinals = new ArrayList<>();
private List<Pair<String, String>> extensionFinals = new ArrayList<>();
+ private List<Pair<String, String>> extensionFinalsAtTheEnd = new
ArrayList<>();
+ private List<Pair<String, String>> extensionMethodsAtTheClassDef = new
ArrayList<>();
private List<List<String>> imports = new ArrayList<>();
private String baseClassName;
private String baseClassDef;
private String optionsBlock;
private boolean read = false;
private boolean shouldReplace = false;
- private String oldWord = null;
- private String newWord = null;
+
+ // Used in @merge replace. If set to true, applies each block's changes
and stores them.
+ private boolean shouldApplyEachBlockChange = true;
+
+ private String oldPhrase = null;
+ private String newPhrase = null;
@Parameter(property = "grammarix.base")
private String base;
@@ -115,12 +128,16 @@
private String parserClassName;
private String lastIdentifier;
+ @Parameter(property = "grammarix.parserClassModifier")
+ private String parserClassModifier;
+
@Override
public void execute() throws MojoExecutionException {
base = new File(base).getAbsolutePath();
getLog().info("Base dir: " + base);
getLog().info("Grammar-base: " + gbase);
getLog().info("Grammar-extension: " + gextension);
+ getLog().info("Output: " + output);
processBase();
processExtension();
generateOutput();
@@ -166,7 +183,29 @@
writer.newLine();
// Class definition
- writer.write(baseClassDef.replaceAll(baseClassName,
parserClassName));
+ String classDef = baseClassDef.replaceAll(baseClassName,
parserClassName);
+ if (parserClassModifier != null && parserClassModifier.length() >
0) {
+ // Adds a class modifier if any.
+ classDef = classDef.replaceFirst(KWCLASS, parserClassModifier
+ " " + KWCLASS);
+ }
+
+ // Process extensions at the end of the class definition
+ if (!extensionMethodsAtTheClassDef.isEmpty()) {
+ int index = classDef.lastIndexOf(CLOSE_BRACE);
+ if (index != -1) {
+ String classDefExtension = "";
+ for (Pair<String, String> element :
extensionMethodsAtTheClassDef) {
+ classDefExtension += toOutput(element.first);
+ classDefExtension += "\n";
+ classDefExtension += element.second;
+ classDefExtension += "\n";
+ }
+ classDef =
+ classDef.substring(0, index) + "\n" +
classDefExtension + "\n" + classDef.substring(index);
+ }
+ }
+
+ writer.write(classDef);
writer.newLine();
// Parser End
@@ -207,6 +246,13 @@
}
for (Pair<String, String> element : baseFinals) {
+ writer.write(toOutput(element.first));
+ writer.newLine();
+ writer.write(element.second);
+ writer.newLine();
+ }
+
+ for (Pair<String, String> element : extensionFinalsAtTheEnd) {
writer.write(toOutput(element.first));
writer.newLine();
writer.write(element.second);
@@ -671,9 +717,15 @@
case NEWPRODUCTION:
nextOperation = NEWPRODUCTION;
break;
+ case NEW_AT_THE_END_PRODUCTION:
+ nextOperation = NEW_AT_THE_END_PRODUCTION;
+ break;
+ case NEW_AT_THE_END_CLASS_DEFINITION:
+ nextOperation = NEW_AT_THE_END_CLASS_DEFINITION;
+ break;
case MERGEPRODUCTION:
nextOperation = MERGEPRODUCTION;
- shouldReplace = shouldReplace(tokens);
+ shouldReplace = shouldReplace(tokens,
position.line);
break;
case OVERRIDEPRODUCTION:
nextOperation = OVERRIDEPRODUCTION;
@@ -693,6 +745,10 @@
case NEWPRODUCTION:
handleNew(identifier, reader);
break;
+ case NEW_AT_THE_END_CLASS_DEFINITION:
+ readFinalProduction(identifier, reader);
+ addFinalProduction(identifier,
extensionMethodsAtTheClassDef);
+ break;
case OVERRIDEPRODUCTION:
handleOverride(identifier, reader);
break;
@@ -704,12 +760,16 @@
}
nextOperation = NEWPRODUCTION;
} else if (openAngularIndex == 0) {
- if (nextOperation != NEWPRODUCTION) {
+ if (nextOperation != NEWPRODUCTION && nextOperation !=
NEW_AT_THE_END_PRODUCTION) {
throw new MojoExecutionException("Can only add new
REGEX production kind");
}
position.index = position.line.indexOf(OPEN_ANGULAR);
readFinalProduction(identifier, reader);
- addFinalProduction(identifier, extensionFinals);
+ if (nextOperation == NEWPRODUCTION) {
+ addFinalProduction(identifier, extensionFinals);
+ } else if (nextOperation == NEW_AT_THE_END_PRODUCTION) {
+ addFinalProduction(identifier,
extensionFinalsAtTheEnd);
+ }
} else if (identifier.length() > 0 ||
position.line.trim().length() > 0) {
identifier.append(position.line);
identifier.append('\n');
@@ -721,16 +781,64 @@
}
}
- private boolean shouldReplace(String[] tokens) throws
MojoExecutionException {
+ private boolean shouldReplace(String[] tokens, String currentLine) throws
MojoExecutionException {
boolean replace = false;
- if (tokens.length == 5) {
- if (tokens[1].equals(REPLACE) && tokens[3].equals(WITH)) {
- shouldReplace = true;
- oldWord = tokens[2];
- newWord = tokens[4];
- } else {
- throw new MojoExecutionException("Allowed syntax after @merge:
<REPLACE> oldWord <WITH> newWord");
+ String errMessage = "Allowed syntax after @merge: <REPLACE>
\"oldPhrase\" <WITH> \"newPhrase\" <TRUE|FALSE>.";
+ String errMessage1 = "The old phrase should be place between two
quotes. (E.g., \"old\")";
+ String errMessage2 = "The new phrase should be place between two
quotes. (E.g., \"new\")";
+ // @merge replace "oldphrase" with "newphrase" proceedBlock:true/false
+ if (tokens.length >= 6) {
+ // Checks whether "replace" exists.
+ if (!tokens[1].equalsIgnoreCase(REPLACE)) {
+ throw new MojoExecutionException(errMessage);
}
+
+ // Checks whether "with" exists.
+ boolean withFound = false;
+ for (int i = 3; i < tokens.length; i++) {
+ if (tokens[i].equalsIgnoreCase(WITH)) {
+ withFound = true;
+ break;
+ }
+ }
+ if (!withFound) {
+ throw new MojoExecutionException(errMessage);
+ }
+
+ // Check whether the last parameter is true/false.
+ // If this is true, then we check all blocks and process before:
and after:.
+ // If not, we don't process all blocks and replace "old" with
"new" for all instances for
+ // the entire part of the given method.
+ if (tokens[tokens.length - 1].equalsIgnoreCase(OPTION_TRUE)) {
+ shouldApplyEachBlockChange = true;
+ } else if (tokens[tokens.length -
1].equalsIgnoreCase(OPTION_FALSE)) {
+ shouldApplyEachBlockChange = false;
+ } else {
+ throw new MojoExecutionException(errMessage);
+ }
+
+ // Gets the old phrase.
+ int oldStart = findQuotePos(currentLine, 0);
+ if (oldStart < 0) {
+ throw new MojoExecutionException(errMessage1);
+ }
+ int oldEnd = findQuotePos(currentLine, oldStart + 1);
+ if (oldEnd < 0) {
+ throw new MojoExecutionException(errMessage1);
+ }
+ oldPhrase = currentLine.substring(oldStart + 1, oldEnd);
+
+ // Gets the new phrase.
+ int newStart = findQuotePos(currentLine, oldEnd + 1);
+ if (newStart < 0) {
+ throw new MojoExecutionException(errMessage2);
+ }
+ int newEnd = findQuotePos(currentLine, newStart + 1);
+ if (newEnd < 0) {
+ throw new MojoExecutionException(errMessage2);
+ }
+ newPhrase = currentLine.substring(newStart + 1, newEnd);
+ replace = true;
}
return replace;
}
@@ -776,26 +884,37 @@
throw new MojoExecutionException(identifier.toString() + " doesn't
exist in base grammar");
} else if (shouldReplace) {
Pair<String, String> baseMethods = extensibles.get(sig);
- baseMethods.first = baseMethods.first.replaceAll(oldWord, newWord);
- baseMethods.second = baseMethods.second.replaceAll(oldWord,
newWord);
+ // Literally replaces the old phrase with the new phrase.
+ baseMethods.first = stringReplaceAll(baseMethods.first, oldPhrase,
newPhrase);
+ baseMethods.second = stringReplaceAll(baseMethods.second,
oldPhrase, newPhrase);
shouldReplace = false;
}
String[] amendments = new String[6];
- mergeElements.put(sig, amendments);
+ if (shouldApplyEachBlockChange) {
+ // Applies and stores each block's change only if
shouldApplyEachBlockChange is set to true.
+ mergeElements.put(sig, amendments);
+ } else {
+ // If shouldApplyEachBlockChange is false, no result from each
block's change are not stored.
+ shouldApplyEachBlockChange = true;
+ }
// we don't need the identifier anymore
identifier.setLength(0);
+ // The first block
readBlock(reader, OPEN_BRACE, CLOSE_BRACE);
String block = record.toString();
extractBeforeAndAfter(block, amendments, 0, 1);
record.reset();
position.index = 0;
position.line = reader.readLine();
+ // Skips empty lines.
while (position.line != null && position.line.trim().length() == 0) {
position.line = reader.readLine();
}
+ // The second block
int openBraceIndex = position.line.indexOf(OPEN_BRACE);
if (openBraceIndex > -1) {
position.index = openBraceIndex;
+ // Reads the entire part of the second block - two OPEN_BRACE and
two CLOSE_BRACE.
readBlock(reader, OPEN_BRACE, CLOSE_BRACE);
} else {
throw new MojoExecutionException("merge element doesn't have a
second block");
@@ -917,4 +1036,35 @@
}
return aString.toString();
}
+
+ /**
+ * Literally replaces the given string.
+ */
+ private String stringReplaceAll(String baseStr, String oldPhrase, String
newPhrase) {
+ int index = baseStr.indexOf(oldPhrase);
+ String tempStr = "";
+ while (index < baseStr.length()) {
+ if (index < 0) {
+ return baseStr;
+ }
+ tempStr = baseStr.substring(index, index + oldPhrase.length());
+ if (tempStr.equals(oldPhrase)) {
+ baseStr = baseStr.substring(0, index) + newPhrase +
baseStr.substring(index + oldPhrase.length());
+ }
+ index = baseStr.indexOf(oldPhrase, index + 1);
+ }
+ return baseStr;
+ }
+
+ /**
+ * Finds and returns the first occurrence index of a quote from
startingIndex.
+ *
+ * @param baseStr
+ * @param startingIndex
+ * @return the substring if found. If not, this returns null.
+ */
+ private int findQuotePos(String baseStr, int startingIndex) {
+ return baseStr.indexOf(ExternalDataConstants.QUOTE, startingIndex);
+ }
+
}
diff --git
a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
index fb143a3..63bd38a 100644
---
a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
+++
b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
@@ -1,14 +1,29 @@
+// If you want to put an additional import, just add import statements like
the following.
+// import package name
import org.apache.asterix.lang.extension.EchoStatement;
-// to remove an import, we use the keyword, unimport
+
+// To remove an import, we use the keyword, unimport
// unimport package name
+
+// If you want to add a method in the class definition (before PARSER_END),
use the following phrase and attach
+// new method right after the phrase.
+// @new_at_the_end
+@new_at_the_class_def
+ public void initScope() {
+ scopeStack.push(RootScopeFactory.createRootScope(this));
+ }
// Merging of non-terminals can only be done on non-terminals which conform to
the following structure.
// Content will simply be prepended or appended to the base blocks.
// Note: refrain from using the strings "before:" and "after:" in the merge
areas as that will break the merge.
// As a workaround, you can always override
-// one additional possible change is direct replacement and it can be done
through
-// @merge replace "baseWord" with "extensionWord"
-
+// one additional possible change is direct replacement and it can be done
through the followin syntax:
+// @merge replace "base phrase" with "extension phrase" true/false
+// Here, true/false tells whether the tool needs to process the three blocks.
+// If true, like normal @merge case, before and after clause in each block
will be processed
+// after "base phrase" in the blocks have been replaced with "new phrase".
+// If false, then it just expects the blank form that consists of three blocks
and not process them.
+// Only, "base phrase" in the blocks will be replaced with "new phrase".
@merge
Statement SingleStatement() throws ParseException:
{
@@ -26,6 +41,24 @@
}
}
+// In the following case, all instances of the first phrase inside of "" will
be replaced with the second phrase in "".
+// Also, we don't check "before:" and "after:" section of each area. That
check will be ignored since
+// the last parameter is set to false.
+@merge replace "nameComponents = QualifiedName() (<AS> var = Variable())?"
with "nameComponents = QualifiedName() (<AS> var = Variable())? " false
+InsertStatement InsertStatement() throws ParseException:
+{
+ // merge area 1
+}
+{
+ (
+ // merge area 2
+ )
+ {
+ // merge area 3
+ }
+}
+
+
// The default
// Adding a new node. if a node exists, it will throw an exception.
@new
@@ -40,11 +73,21 @@
}
}
+// Overriding a non-terminal. if exists in base, it will be overriden,
otherwise, it will be added
+// @override
+
+
+// Terminals can be added like the following.
<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
<ECHO : "echo">
}
-// Overriding a non-terminal. if exists in base, it will be overriden,
otherwise, it will be added
-// @override
+// If something needs to be added at the end of file, you can use
@new_at_the_end like the following.
+@new_at_the_end
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <METAVARIABLE : "$$" <IDENTIFIER> >
+}
diff --git
a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
index 5475fa4..3f929c1 100644
---
a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
+++
b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
@@ -41,6 +41,7 @@
<gextension>src/test/resources/lang/extension.jj</gextension>
<output>target/generated-sources/lang/grammar.jj</output>
<parserClassName>ExtendedParser</parserClassName>
+ <parserClassModifier></parserClassModifier>
<packageName>org.apache.asterix.lang.extension.parser</packageName>
</configuration>
<executions>
--
To view, visit https://asterix-gerrit.ics.uci.edu/1431
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Icafc01ebe591b81b3c3f69ea9da123dbfad19bb0
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Taewoo Kim <wangs...@yahoo.com>
