pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jul...@apache.org
Subject svn commit: r1379210 - in /pig/trunk: CHANGES.txt src/org/apache/pig/parser/QueryParserDriver.java
Date Thu, 30 Aug 2012 23:03:38 GMT
Author: julien
Date: Thu Aug 30 23:03:37 2012
New Revision: 1379210

URL: http://svn.apache.org/viewvc?rev=1379210&view=rev
Log:
PIG-2896  Pig does not fail anymore if two macros are declared with the same name

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1379210&r1=1379209&r2=1379210&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Aug 30 23:03:37 2012
@@ -254,6 +254,8 @@ OPTIMIZATIONS
 
 BUG FIXES
 
+PIG-2896:  Pig does not fail anymore if two macros are declared with the same name (julien)
+
 PIG-2848: TestBuiltInBagToTupleOrString fails now that mock.Storage enforces not overwriting
output (julien)
 
 PIG-2884: JobControlCompiler mis-logs after reducer estimation (billgraham)

Modified: pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java?rev=1379210&r1=1379209&r2=1379210&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java Thu Aug 30 23:03:37 2012
@@ -58,28 +58,28 @@ import org.apache.pig.tools.pigstats.Scr
 
 public class QueryParserDriver {
     private static final Log LOG = LogFactory.getLog(QueryParserDriver.class);
-    
+
     private static final String MACRO_DEF = "MACRO_DEF";
     private static final String MACRO_INLINE = "MACRO_INLINE";
     private static final String IMPORT_DEF = "import";
-    
+
     private PigContext pigContext;
     private String scope;
     private Map<String, String>fileNameMap;
     private Map<String, Operator> operators;
     private Set<String> importSeen;
     private Set<String> macroSeen;
-    
+
     private static Map<String, FetchFileRet> fnameMap = new HashMap<String, FetchFileRet>();
-    
+
     public QueryParserDriver(PigContext pigContext, String scope, Map<String, String>
fileNameMap) {
         this.pigContext = pigContext;
         this.scope = scope;
         this.fileNameMap = fileNameMap;
         importSeen = new HashSet<String>();
-        macroSeen = new HashSet<String>();        
+        macroSeen = new HashSet<String>();
     }
-    
+
     private static Tree parseSchema(CommonTokenStream tokens) throws ParserException {
         QueryParser parser = QueryParserUtils.createParser(tokens);
 
@@ -104,14 +104,14 @@ public class QueryParserDriver {
         CommonTokenStream tokenStream = tokenize( input, null );
         LogicalSchema schema = null;
         Tree ast = parseSchema( tokenStream );
-        
-        try{       
+
+        try{
             CommonTreeNodeStream nodes = new CommonTreeNodeStream( ast );
             AstValidator walker = new AstValidator( nodes );
             ast = (Tree)walker.field_def_list().getTree();
             checkError( walker );
-            
-            LogicalPlanGenerator planGenerator = 
+
+            LogicalPlanGenerator planGenerator =
                 new LogicalPlanGenerator( new CommonTreeNodeStream( ast ), pigContext, scope,
fileNameMap );
             schema = planGenerator.field_def_list().schema;
             checkError( planGenerator );
@@ -120,7 +120,7 @@ public class QueryParserDriver {
         } catch(Exception ex) {
             throw new ParserException( ex.getMessage(), ex );
         }
-        
+
         return schema;
     }
 
@@ -148,14 +148,14 @@ public class QueryParserDriver {
         CommonTokenStream tokenStream = tokenize( input, null );
         Object value = null;
         Tree ast = parseConstant( tokenStream );
-        
-        try{       
+
+        try{
             CommonTreeNodeStream nodes = new CommonTreeNodeStream( ast );
             AstValidator walker = new AstValidator( nodes );
             ast = (Tree)walker.literal().getTree();
             checkError( walker );
 
-            LogicalPlanGenerator planGenerator = 
+            LogicalPlanGenerator planGenerator =
                 new LogicalPlanGenerator( new CommonTreeNodeStream( ast ), pigContext, scope,
fileNameMap );
             value = planGenerator.literal().value;
             checkError( planGenerator );
@@ -164,27 +164,27 @@ public class QueryParserDriver {
         } catch(Exception ex) {
             throw new ParserException( ex.getMessage(), ex );
         }
-        
+
         return value;
     }
 
     public LogicalPlan parse(String query) throws ParserException {
         LogicalPlan plan = null;
-        
+
         ScriptState ss = ScriptState.get();
         CommonTokenStream tokenStream = tokenize(query, ss.getFileName());
-        
+
         Tree ast = parse( tokenStream );
         ast = expandMacro( ast );
 
-        try{       
+        try{
             ast = validateAst( ast );
-            LogicalPlanGenerator planGenerator = 
+            LogicalPlanGenerator planGenerator =
                 new LogicalPlanGenerator( new CommonTreeNodeStream( ast ), pigContext, scope,
fileNameMap );
             planGenerator.query();
-            
+
             checkError( planGenerator );
-            
+
             plan = planGenerator.getLogicalPlan();
             operators = planGenerator.getOperators();
         } catch(RecognitionException ex) {
@@ -192,10 +192,10 @@ public class QueryParserDriver {
         } catch(Exception ex) {
             throw new ParserException( ex.getMessage(), ex );
         }
-        
+
         return plan;
     }
-        
+
     public Map<String, Operator> getOperators() {
         return operators;
     }
@@ -214,7 +214,7 @@ public class QueryParserDriver {
         checkError(lexer);
         return tokens;
     }
-    
+
     private static void checkError(BaseRecognizer recognizer)
             throws ParserException {
         int errorCount = recognizer.getNumberOfSyntaxErrors();
@@ -235,25 +235,25 @@ public class QueryParserDriver {
             throw new ParserException(msg);
         } catch(RuntimeException ex) {
             throw new ParserException( ex.getMessage() );
-        }          
+        }
 
         Tree ast = (Tree) result.getTree();
         checkError(parser);
 
         return ast;
     }
-    
+
     private static Tree validateAst(Tree ast) throws RecognitionException, ParserException
{
         CommonTreeNodeStream nodes = new CommonTreeNodeStream( ast );
         AstValidator walker = new AstValidator( nodes );
         AstValidator.query_return newResult = walker.query();
         Tree newAst = (Tree)newResult.getTree();
-        
+
         checkError( walker );
-        
+
         return newAst;
     }
-    
+
     Tree expandMacro(Tree ast) throws ParserException {
         LOG.debug("Original macro AST:\n" + ast.toStringTree() + "\n");
 
@@ -284,16 +284,16 @@ public class QueryParserDriver {
 
         return ast;
     }
-     
+
     private void inlineMacro(List<CommonTree> inlineNodes,
             List<PigMacro> macroDefs) throws ParserException {
         for (CommonTree t : inlineNodes) {
             Set<String> macroStack = new HashSet<String>();
             CommonTree newTree = PigMacro.macroInline(t, macroDefs, macroStack);
-            
+
             List<CommonTree> nodes = new ArrayList<CommonTree>();
             traverseInline(newTree, nodes);
-   
+
             if (nodes.isEmpty()) {
                 QueryParserUtils.replaceNodeWithNodeList(t, newTree, null);
             } else {
@@ -301,7 +301,7 @@ public class QueryParserDriver {
             }
         }
     }
-    
+
     private void traverseInline(Tree t, List<CommonTree> nodes) {
         if (t.getText().equals(MACRO_INLINE)) {
             nodes.add((CommonTree)t);
@@ -310,21 +310,21 @@ public class QueryParserDriver {
         for (int i = 0; i < n; i++) {
             Tree t0 = t.getChild(i);
             traverseInline(t0, nodes);
-        }       
+        }
     }
-    
+
     private boolean expandImport(Tree ast) throws ParserException {
         List<CommonTree> nodes = new ArrayList<CommonTree>();
         traverseImport(ast, nodes);
         if (nodes.isEmpty()) return false;
-        
+
         for (CommonTree t : nodes) {
             macroImport(t);
         }
-        
+
         return true;
     }
-    
+
     static void traverseImport(Tree t, List<CommonTree> nodes) {
         if (t.getText().equalsIgnoreCase(IMPORT_DEF)) {
             nodes.add((CommonTree)t);
@@ -335,11 +335,11 @@ public class QueryParserDriver {
             traverseImport(t0, nodes);
         }
     }
-    
+
     static void traverse(Tree t, List<CommonTree> macroNodes,
             List<CommonTree> inlineNodes) {
         if (t.getText().equals(MACRO_DEF)) {
-            macroNodes.add((CommonTree) t.getParent());  
+            macroNodes.add((CommonTree) t.getParent());
         } else if (t.getText().equals(MACRO_INLINE)) {
             inlineNodes.add((CommonTree) t);
         }
@@ -349,7 +349,7 @@ public class QueryParserDriver {
             traverse(t0, macroNodes, inlineNodes);
         }
     }
-    
+
     private FetchFileRet getMacroFile(String fname) {
         FetchFileRet localFileRet = null;
         try {
@@ -365,11 +365,11 @@ public class QueryParserDriver {
                 // ignore this since we'll attempt to load as a resource before failing
                 LOG.debug(String.format("Macro file %s was not found", fname));
               }
-              
+
               // try loading the macro file as a resource in case it is packaged in a registered
jar
               if (localFileRet == null) {
                 LOG.debug(String.format("Attempting to load macro file %s as a resource",
fname));
-                
+
                 try
                 {
                   localFileRet = FileLocalizer.fetchResource(fname);
@@ -382,7 +382,7 @@ public class QueryParserDriver {
                   throw new ExecException("file '" + fname + "' does not exist.", 101, PigException.INPUT);
                 }
               }
-                            
+
               fnameMap.put(fname, localFileRet);
             }
         } catch (IOException e) {
@@ -390,7 +390,7 @@ public class QueryParserDriver {
         }
         return localFileRet;
     }
-    
+
     /*
      * MacroDef node has two child nodes:
      *      1. name
@@ -399,13 +399,11 @@ public class QueryParserDriver {
     private PigMacro makeMacroDef(CommonTree t, Map<String, PigMacro> seen)
             throws ParserException {
         String mn = t.getChild(0).getText();
-        
+
         if (!macroSeen.add(mn)) {
             String msg = getErrorMessage(null, t, null,
                     "Duplicated macro name '" + mn + "'");
-            LOG.warn(msg);
-            // Disabling this exception, see https://issues.apache.org/jira/browse/PIG-2279
-            // throw new ParserException(msg);
+            throw new ParserException(msg);
         }
 
         if (seen != null) {
@@ -413,13 +411,13 @@ public class QueryParserDriver {
                 macroSeen.add(s);
             }
         }
-        
+
         String fname = ((PigParserNode)t).getFileName();
-        
+
         Tree defNode = t.getChild(1);
 
         // get parameter markers
-        ArrayList<String> params = new ArrayList<String>(); 
+        ArrayList<String> params = new ArrayList<String>();
         Tree paramNode = defNode.getChild(0);
         int n = paramNode.getChildCount();
         for (int i = 0; i < n; i++) {
@@ -427,7 +425,7 @@ public class QueryParserDriver {
         }
 
         // get return alias markers
-        ArrayList<String> returns = new ArrayList<String>(); 
+        ArrayList<String> returns = new ArrayList<String>();
         Tree retNode = defNode.getChild(1);
         int m = retNode.getChildCount();
         for (int i = 0; i < m; i++) {
@@ -448,7 +446,7 @@ public class QueryParserDriver {
         }
 
         PigMacro pm = new PigMacro(mn, fname, params, returns, body, seen);
-        
+
         try {
             pm.validate();
         } catch (IOException e) {
@@ -456,11 +454,11 @@ public class QueryParserDriver {
                     "Invalid macro definition: ", e.getMessage());
             throw new ParserException(msg);
         }
-        
-        // set the starting line number of the macro 
+
+        // set the starting line number of the macro
         PigParserNode pnode = (PigParserNode)bodyNode.getChild(0);
         pm.setStartLine(pnode.getStartLine());
-        
+
         seen.put(mn, pm);
 
         // delete this node
@@ -470,7 +468,7 @@ public class QueryParserDriver {
 
         return pm;
     }
-        
+
     private void macroImport(CommonTree t) throws ParserException {
         // remove quote
         String fname = t.getChild(0).getText();
@@ -482,9 +480,9 @@ public class QueryParserDriver {
             // Disabling this exception, see https://issues.apache.org/jira/browse/PIG-2279
             // throw new ParserException(msg);
         }
-        
+
         FetchFileRet localFileRet = getMacroFile(fname);
-        
+
         BufferedReader in = null;
         try {
             in = new BufferedReader(new FileReader(localFileRet.file));
@@ -493,7 +491,7 @@ public class QueryParserDriver {
                     "Failed to import file '" + fname + "'", e.getMessage());
             throw new ParserException(msg);
         }
-        
+
         StringBuilder sb = new StringBuilder();
         String line = null;
         try {
@@ -507,20 +505,20 @@ public class QueryParserDriver {
                     "Failed to read file '" + fname + "'", e.getMessage());
             throw new ParserException(msg);
         }
-        
+
         // parse
         CommonTokenStream tokenStream = tokenize(sb.toString(), fname);
-        
+
         Tree ast = null;
         try {
             ast = parse( tokenStream );
         } catch(RuntimeException ex) {
             throw new ParserException( ex.getMessage() );
-        } 
-        
+        }
+
         QueryParserUtils.replaceNodeWithNodeList(t, (CommonTree)ast, fname);
     }
-    
+
     private String getErrorMessage(String importFile,
             CommonTree t, String header, String reason) {
         StringBuilder sb = new StringBuilder();
@@ -539,5 +537,5 @@ public class QueryParserDriver {
             sb.append(". Reason: ").append(reason);
         }
         return sb.toString();
-    }   
+    }
 }



Mime
View raw message