pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject svn commit: r692253 [3/3] - in /incubator/pig/branches/types: ./ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/ src/org/apache/pig/backend/hadoop/ex...
Date Thu, 04 Sep 2008 21:25:43 GMT
Modified: incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
(original)
+++ incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
Thu Sep  4 14:25:41 2008
@@ -188,6 +188,7 @@
 <PIG_START> MORE :
 {
 	<"'"> {prevState = PIG_START;} : IN_STRING
+|	<"`"> {prevState = PIG_START;} : IN_COMMAND
 |	<["A","a"]["S","s"]> {prevState = PIG_START;} : SCHEMA_DEFINITION
 |   <["G","g"]["E","e"]["N","n"]["E","e"]["R","r"]["A","a"]["T","t"]["E","e"]> {prevState
= PIG_START;} : GENERATE
 |       <"{"> {pigBlockLevel = 1;} : IN_BLOCK
@@ -220,6 +221,14 @@
 |	<(~[])>
 }
 
+<IN_COMMAND> MORE :
+{
+	<"\\`">
+|	<"`"> { SwitchTo(prevState);}
+|	<("\n" | "\r" | "\r\n")> {secondary_prompt();}
+|	<(~[])>
+}
+
 <GENERATE> MORE :
 {
     <"{"> 
@@ -282,6 +291,7 @@
 |	<"{"> {pigBlockLevel++;}
 |       <"}"(";")?> {pigBlockLevel--; if (pigBlockLevel == 0) SwitchTo(PIG_END);}
 |	<"'"> {prevState = IN_BLOCK;} : IN_STRING
+|	<"`"> {prevState = IN_BLOCK;} : IN_COMMAND
 |	<"--"> {prevState = IN_BLOCK;} : SINGLE_LINE_COMMENT
 |	<"/*"> {prevState = IN_BLOCK;} : MULTI_LINE_COMMENT
 |	<("\n" | "\r" | "\r\n")> {secondary_prompt();}

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestInputOutputFileValidator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestInputOutputFileValidator.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestInputOutputFileValidator.java
(original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestInputOutputFileValidator.java
Thu Sep  4 14:25:41 2008
@@ -115,7 +115,7 @@
             new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
         FileSpec filespec2 =
             new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
-        LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, null) ;       
+        LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, null, true) ;  
    
         LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2) ;
         
         plan.add(load) ;

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestJobSubmission.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestJobSubmission.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestJobSubmission.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestJobSubmission.java Thu Sep 
4 14:25:41 2008
@@ -153,7 +153,7 @@
         FileSpec LFSpec = new FileSpec(hadoopLdFile,new FuncSpec(PigStorage.class.getName()));
         FileSpec SFSpec = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName()));
 
-        POLoad ld = new POLoad(new OperatorKey("", r.nextLong()));
+        POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
         POStore st = new POStore(new OperatorKey("", r.nextLong()));
         ld.setPc(pc);
         ld.setLFile(LFSpec);
@@ -200,7 +200,7 @@
         FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName()));
         FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
 
-        POLoad ld = new POLoad(new OperatorKey("", r.nextLong()));
+        POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
         POStore st = new POStore(new OperatorKey("", r.nextLong()));
         ld.setPc(pc);
         ld.setLFile(LFSpec);
@@ -247,7 +247,7 @@
         FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName()));
         FileSpec SFSpec = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName()));
 
-        POLoad ld = new POLoad(new OperatorKey("", r.nextLong()));
+        POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
         POStore st = new POStore(new OperatorKey("", r.nextLong()));
         ld.setPc(pc);
         ld.setLFile(LFSpec);
@@ -309,7 +309,7 @@
         FileSpec LFSpec = new FileSpec(hadoopLdFile,new FuncSpec(PigStorage.class.getName()));
         FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
 
-        POLoad ld = new POLoad(new OperatorKey("", r.nextLong()));
+        POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
         POStore st = new POStore(new OperatorKey("", r.nextLong()));
         ld.setPc(pc);
         ld.setLFile(LFSpec);
@@ -376,7 +376,7 @@
         FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName(),
new String[]{","}));
         FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
 
-        POLoad ld = new POLoad(new OperatorKey("", r.nextLong()));
+        POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
         POStore st = new POStore(new OperatorKey("", r.nextLong()));
         ld.setPc(pc);
         ld.setLFile(LFSpec);

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestLocalJobSubmission.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLocalJobSubmission.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestLocalJobSubmission.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestLocalJobSubmission.java Thu
Sep  4 14:25:41 2008
@@ -373,7 +373,7 @@
         FileSpec LFSpec = new FileSpec(ldFile, new FuncSpec(PigStorage.class.getName(), new
String[]{","}));
         FileSpec SFSpec = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName()));
 
-        POLoad ld = new POLoad(new OperatorKey("", r.nextLong()));
+        POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
         POStore st = new POStore(new OperatorKey("", r.nextLong()));
         ld.setPc(pc);
         ld.setLFile(LFSpec);

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java Thu
Sep  4 14:25:41 2008
@@ -896,11 +896,11 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
         LOLoad load2 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // schema for input#1
         Schema inputSchema1 = null ;
@@ -1020,11 +1020,11 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
         LOLoad load2 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // schema for input#1
         Schema inputSchema1 = null ;
@@ -1192,7 +1192,7 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // schema for input#1
         Schema inputSchema1 = null ;
@@ -1260,7 +1260,7 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // schema for input#1
         Schema inputSchema1 = null ;
@@ -1330,7 +1330,7 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // schema for input#1
         Schema inputSchema1 = null ;
@@ -1400,7 +1400,7 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // schema for input#1
         Schema inputSchema1 = null ;
@@ -1480,7 +1480,7 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // schema for input#1
         Schema inputSchema1 = null ;
@@ -1578,7 +1578,7 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // schema for input#1
         Schema inputSchema1 = null ;
@@ -1673,7 +1673,7 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // schema for input#1
         Schema inputSchema1 = null ;
@@ -1801,7 +1801,7 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // schema for input#1
         Schema inputSchema1 = null ;
@@ -1905,11 +1905,11 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
         LOLoad load2 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // schema for input#1
         Schema inputSchema1 = null ;
@@ -2076,11 +2076,11 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
         LOLoad load2 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // schema for input#1
         Schema inputSchema1 = null ;
@@ -2215,11 +2215,11 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
         LOLoad load2 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // schema for input#1
         Schema inputSchema1 = null ;

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java
(original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java
Thu Sep  4 14:25:41 2008
@@ -445,7 +445,7 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // set schemas
         load1.setEnforcedSchema(null) ;
@@ -511,7 +511,7 @@
         LOLoad load1 = new LOLoad(plan,
                                   genNewOperatorKey(),
                                   new FileSpec("pi", new FuncSpec(pigStorage)),
-                                  null) ;
+                                  null, true) ;
 
         // set schemas
         load1.setEnforcedSchema(null) ;

Modified: incubator/pig/branches/types/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/Util.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/Util.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/Util.java Thu Sep  4 14:25:41 2008
@@ -17,9 +17,14 @@
  */
 package org.apache.pig.test;
 
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import static java.util.regex.Matcher.quoteReplacement;
+import junit.framework.Assert;
 
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.data.*;
@@ -145,6 +150,45 @@
         return t;
     }
 
+    /**
+     * Helper to create a temporary file with given input data for use in test cases.
+     *  
+     * @param tmpFilenamePrefix file-name prefix
+     * @param tmpFilenameSuffix file-name suffix
+     * @param inputData input for test cases, each string in inputData[] is written
+     *                  on one line
+     * @return {@link File} handle to the created temporary file
+     * @throws IOException
+     */
+	static public File createInputFile(String tmpFilenamePrefix, 
+			                           String tmpFilenameSuffix, 
+			                           String[] inputData) 
+	throws IOException {
+		File f = File.createTempFile(tmpFilenamePrefix, tmpFilenameSuffix);
+		PrintWriter pw = new PrintWriter(f);
+		for (int i=0; i<inputData.length; i++){
+			pw.println(inputData[i]);
+		}
+		pw.close();
+		return f;
+	}
+
+	/**
+	 * Helper function to check if the result of a Pig Query is in line with 
+	 * expected results.
+	 * 
+	 * @param actualResults Result of the executed Pig query
+	 * @param expectedResults Expected results to validate against
+	 */
+	static public void checkQueryOutputs(Iterator<Tuple> actualResults, 
+			                        Tuple[] expectedResults) {
+	    
+		for (Tuple expected : expectedResults) {
+			Tuple actual = actualResults.next();
+			Assert.assertEquals(expected.toString(), actual.toString());
+		}
+	}
+	
 	static public void printQueryOutput(Iterator<Tuple> actualResults, 
                Tuple[] expectedResults) {
 
@@ -160,4 +204,18 @@
         }
         System.out.println("---End----") ;
     }
+
+	/**
+     * Helper method to replace all occurrences of "\" with "\\" in a 
+     * string. This is useful to fix the file path string on Windows
+     * where "\" is used as the path separator.
+     * 
+     * @param str Any string
+     * @return The resulting string
+     */
+	public static String encodeEscape(String str) {
+	    String regex = "\\\\";
+	    String replacement = quoteReplacement("\\\\");
+	    return str.replaceAll(regex, replacement);
+	}
 }

Modified: incubator/pig/branches/types/test/org/apache/pig/test/utils/GenPhyOp.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/GenPhyOp.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/GenPhyOp.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/GenPhyOp.java Thu Sep  4 14:25:41
2008
@@ -568,7 +568,7 @@
 
     
     public static POLoad topLoadOp() {
-        POLoad ret = new POLoad(new OperatorKey("", r.nextLong()));
+        POLoad ret = new POLoad(new OperatorKey("", r.nextLong()), true);
         ret.setPc(pc);
         return ret;
     }

Modified: incubator/pig/branches/types/test/org/apache/pig/test/utils/TestHelper.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/TestHelper.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/TestHelper.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/TestHelper.java Thu Sep  4
14:25:41 2008
@@ -170,11 +170,11 @@
     public static boolean areFilesSame(FileSpec expLocal, FileSpec actHadoop, PigContext
pc) throws ExecException, IOException{
         Random r = new Random();
         
-        POLoad ldExp = new POLoad(new OperatorKey("", r.nextLong()));
+        POLoad ldExp = new POLoad(new OperatorKey("", r.nextLong()), true);
         ldExp.setPc(pc);
         ldExp.setLFile(expLocal);
         
-        POLoad ldAct = new POLoad(new OperatorKey("", r.nextLong()));
+        POLoad ldAct = new POLoad(new OperatorKey("", r.nextLong()), true);
         ldAct.setPc(pc);
         ldAct.setLFile(actHadoop);
         

Modified: incubator/pig/branches/types/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java
(original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java
Thu Sep  4 14:25:41 2008
@@ -42,7 +42,7 @@
             LOLoad load = new LOLoad(plan,
                                       genNewOperatorKey(),
                                       new FileSpec("pi", new FuncSpec(pigStorage)),
-                                      null) ;
+                                      null, true) ;
             return load ;
         } catch (IOException e) {
             throw new AssertionError("This cannot happen") ;

Modified: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java
(original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java
Thu Sep  4 14:25:41 2008
@@ -94,7 +94,7 @@
         FileSpec fileSpec = new FileSpec("pi",
                                          new FuncSpec(PigStorage.class.getName())) ;
         try {
-            load = new LOLoad(plan, getKey(node.attributes), fileSpec, null) ;
+            load = new LOLoad(plan, getKey(node.attributes), fileSpec, null, true) ;
             fillSchema(load, node.attributes) ;
         }
         catch (IOException ioe) {



Mime
View raw message