Author: szita
Date: Thu Apr 23 11:25:37 2020
New Revision: 1876880

URL: http://svn.apache.org/viewvc?rev=1876880&view=rev
Log:
PIG-5362: Parameter substitution of shell cmd results doesn't handle backslash 
addendum (szita)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj
    pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj

Modified: pig/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1876880&r1=1876879&r2=1876880&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Apr 23 11:25:37 2020
@@ -100,6 +100,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-5362: Parameter substitution of shell cmd results doesn't handle backslash 
addendum (szita)
+
 PIG-5395: Pig build is failing due to maven repo access point change (szita)
 
 PIG-5375: NullPointerException for multi-level self unions with Tez 
UnionOptimizer (knoguchi)

Modified: pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj
URL: 
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj?rev=1876880&r1=1876879&r2=1876880&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj (original)
+++ pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj Thu Apr 23 
11:25:37 2020
@@ -43,16 +43,6 @@ public class ParamLoader {
     public void setContext(PreprocessorContext pc) {
         this.pc = pc;
     }
-
-    private static String unquote(String s)
-    {
-        if (s.charAt(0) == '\'' && s.charAt(s.length()-1) == '\'')
-            return s.substring(1, s.length()-1);
-        else if (s.charAt(0) == '"' && s.charAt(s.length()-1) == '"')
-            return s.substring(1, s.length()-1);
-        else
-            return s;
-    }
 }
 
 PARSER_END(ParamLoader)
@@ -81,16 +71,52 @@ TOKEN :
     |
     <OTHER: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " , 
"\n" , "\t" , "\r"] (~["\n","\r"])* >
     |
-    <LITERAL: ("\"" ((~["\""])*("\\\"")?)* "\"")|("'" ((~["'"])*("\\\'")?)* 
"'") >
-    |
     <SHELLCMD: "`" (~["`"])* "`" >
     |
     <EQUALS: "="  >
     |
     <COMMENT: "#" (~["\n"])* ("\n"|"\r")  >
+}
+
+MORE :
+{
+    <DOUBLE_QUOTE_START: ("\"") > : DOUBLE_QUOTE
+    |
+    <SINGLE_QUOTE_START: ("'") > : SINGLE_QUOTE
+}
+
+<DOUBLE_QUOTE> TOKEN :
+{
+    <DOUBLE_QUOTE_LITERAL: ("\"") > {
+        image.deleteCharAt(image.length()-1);
+        image.deleteCharAt(0);
+        matchedToken.image = image.toString();
+    } : DEFAULT
+} 
 
+<SINGLE_QUOTE> TOKEN :
+{
+    <SINGLE_QUOTE_LITERAL: ("'") > {
+        image.deleteCharAt(image.length()-1);
+        image.deleteCharAt(0);
+        matchedToken.image = image.toString();
+    } : DEFAULT 
+}
+
+<DOUBLE_QUOTE > MORE :
+{
+    <ESCAPED_DOUBLE_QUOTE: ("\\\"")> { image.replace(image.length()-2, 
image.length(), "\""); }
+}
+
+<SINGLE_QUOTE > MORE :
+{
+    <ESCAPED_SINGLE_QUOTE: ("\\'")> { image.replace(image.length()-2, 
image.length(), "'"); }
 }
 
+<DOUBLE_QUOTE,SINGLE_QUOTE> MORE:
+{
+    < (~[]) >
+}
 
 
 boolean Parse() throws IOException  :
@@ -117,7 +143,10 @@ boolean Parse() throws IOException  :
                 |
                 val=<SHELLCMD>  { pc.processShellCmd(id.image , val.image);}
                 |
-                val=<LITERAL> { s = unquote(val.image); 
pc.processOrdLine(id.image , s); }
+                val=<SINGLE_QUOTE_LITERAL> { pc.processOrdLine(id.image , 
val.image); }
+                |
+                val=<DOUBLE_QUOTE_LITERAL> { pc.processOrdLine(id.image , 
val.image); }
+            
             )
         )
         |

Modified: pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj
URL: 
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj?rev=1876880&r1=1876879&r2=1876880&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj (original)
+++ pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj Thu Apr 23 
11:25:37 2020
@@ -223,25 +223,22 @@ TOKEN_MGR_DECLS : {
     }: DEFAULT
 }
 
-TOKEN : 
+< DEFAULT, IN_DECLARE > TOKEN : 
 {
-    <NEWLINE: "\n" | "\r">
+    <NEWLINE: "\n" | "\r"> : DEFAULT
     |    
     <SPACE: " " | "\t">
     |
     <WS: "\n" | "\r" | " " | "\t">
-}
-
-// comments(single line and multi-line)
-TOKEN : 
-{
-   <COMMENT: "--"(~["\r","\n"])*
+    |   
+    // comments(single line and multi-line)
+    <COMMENT: "--"(~["\r","\n"])*
           |  "#!" (~["\r","\n"])*
           | "/*" (~["*"])* "*" ("*" | (~["*","/"] (~["*"])* "*"))* "/"
-   >   
+    >   
 }
 
-TOKEN:
+< DEFAULT, IN_DECLARE >TOKEN:
 {
     <#LETTER : ["a"-"z", "A"-"Z"] >
     |
@@ -254,22 +251,31 @@ TOKEN:
 
 TOKEN :
 {
-    <DECLARE: "%declare" >
+    <DECLARE: "%declare" > : IN_DECLARE
     |
-    <PIGDEFAULT: "%default" > 
+    <PIGDEFAULT: "%default" > : IN_DECLARE
+    |
+    <REGISTER: "register"> : IN_REGISTER
+    |
+    <LITERAL: ("\"" ((~["\""])*("\\\"")?)* "\"")|("'" ((~["'"])*("\\\'")?)* 
"'") >
 }
 
-
-TOKEN : 
+< DEFAULT, IN_DECLARE > TOKEN:
 {
-    <REGISTER: "register"> : IN_REGISTER
-    |
     <IDENTIFIER: (<SPECIALCHAR>)*<LETTER>(<DIGIT> | <LETTER> | <SPECIALCHAR>)*>
     |
-    <LITERAL: ("\"" ((~["\""])*("\\\"")?)* "\"")|("'" ((~["'"])*("\\\'")?)* 
"'") >
-    |
-    <SHELLCMD: "`" (~["`"])* "`" >
+    <SHELLCMD: "`" (~["`"])* "`" > : DEFAULT
+}
+
+< IN_DECLARE > MORE :
+{
+    <DOUBLE_QUOTE_START: ("\"") > : DOUBLE_QUOTE
     |
+    <SINGLE_QUOTE_START: ("'") > : SINGLE_QUOTE
+}
+
+< DEFAULT, IN_DECLARE > TOKEN:
+{
     // see others() rule for use of OTHER and NOT_OTHER_CHAR
     // others() is supposed to match 'everything else'. To ensure that others()
     // don't swallow other(all the ones above) tokens, it uses two tokens 
OTHER and NOT_OTHER_CHAR
@@ -281,6 +287,39 @@ TOKEN :
     <NOT_OTHER_CHAR: ["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , 
" " , "\n" , "\t" , "\r", "%", "/", "-", "$"] >
 }
 
+< DOUBLE_QUOTE > TOKEN :
+{
+    <DOUBLE_QUOTE_LITERAL: ("\"") > {
+        image.deleteCharAt(image.length()-1);
+        image.deleteCharAt(0);
+        matchedToken.image = image.toString();
+    } : DEFAULT
+} 
+
+< SINGLE_QUOTE > TOKEN :
+{
+    <SINGLE_QUOTE_LITERAL: ("'") > {
+        image.deleteCharAt(image.length()-1);
+        image.deleteCharAt(0);
+        matchedToken.image = image.toString();
+    } : DEFAULT 
+}
+
+< DOUBLE_QUOTE > MORE :
+{
+    <ESCAPED_DOUBLE_QUOTE: ("\\\"")> { image.replace(image.length()-2, 
image.length(), "\""); }
+}
+
+< SINGLE_QUOTE > MORE :
+{
+    <ESCAPED_SINGLE_QUOTE: ("\\'")> { image.replace(image.length()-2, 
image.length(), "'"); }
+}
+
+< DOUBLE_QUOTE, SINGLE_QUOTE > MORE:
+{
+    < (~[]) >
+}
+
 <IN_REGISTER> MORE : { " " |  "\t" | "\r" | "\n"}
 
 <IN_REGISTER> TOKEN: {
@@ -373,8 +412,9 @@ void param_value(boolean overwrite) thro
             |
             val=<SHELLCMD>  { pc.processShellCmd(id.image , val.image, 
overwrite); }
             |
-            val=<LITERAL> { s = unquote(val.image); 
pc.processOrdLine(id.image, s, overwrite); }
-
+            val=<SINGLE_QUOTE_LITERAL> { pc.processOrdLine(id.image, 
val.image, overwrite); }
+            |
+            val = <DOUBLE_QUOTE_LITERAL> { pc.processOrdLine(id.image, 
val.image, overwrite); }
         )
 }
 


Reply via email to