pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject svn commit: r1362673 - in /pig/trunk: CHANGES.txt test/org/apache/pig/test/TestBuiltin.java test/org/apache/pig/test/TestJoin.java test/org/apache/pig/test/TestPigScriptParser.java test/org/apache/pig/test/TestUnion.java
Date Tue, 17 Jul 2012 21:42:39 GMT
Author: gdfm
Date: Tue Jul 17 21:42:38 2012
New Revision: 1362673

URL: http://svn.apache.org/viewvc?rev=1362673&view=rev
Log:
PIG-2797: Tests should not create their own file URIs through string concatenation, should
use Util.generateURI instead (jgordon via azaroth)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/test/org/apache/pig/test/TestBuiltin.java
    pig/trunk/test/org/apache/pig/test/TestJoin.java
    pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java
    pig/trunk/test/org/apache/pig/test/TestUnion.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1362673&r1=1362672&r2=1362673&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jul 17 21:42:38 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
 
 IMPROVEMENTS
 
+PIG-2797: Tests should not create their own file URIs through string concatenation, should
use Util.generateURI instead (jgordon via azaroth)
+
 PIG-2820: relToAbsolutePath is not replayed properly when Grunt reparses the script after
PIG-2699 (julien)
 
 PIG-2763: Groovy UDFs (herberts via julien)

Modified: pig/trunk/test/org/apache/pig/test/TestBuiltin.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestBuiltin.java?rev=1362673&r1=1362672&r2=1362673&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestBuiltin.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestBuiltin.java Tue Jul 17 21:42:38 2012
@@ -165,7 +165,7 @@ public class TestBuiltin {
     @Before
     public void setUp() throws Exception {
 
-        pigServer = new PigServer(ExecType.LOCAL, new Properties());
+        pigServer = new PigServer(ExecType.LOCAL, cluster.getProperties());
         pigServer.setValidateEachStatement(true);
         // First set up data structs for "base" SUM, MIN and MAX and AVG.
         // The allowed input and expected output data structs for
@@ -870,7 +870,7 @@ public class TestBuiltin {
     public void testCount_ValidNumberOfArguments_WithoutInputSchema_One() throws Exception
{
          File inputFile = createCountInputFile();
          try {
-             pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "';");
+             pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath())
+ "';");
              pigServer.registerQuery("B = group A all;");
              pigServer.registerQuery("C = foreach B generate COUNT(A.$0);");
              assertValidCount();
@@ -885,7 +885,7 @@ public class TestBuiltin {
     public void testCount_ValidNumberOfArguments_WithoutInputSchema_Two() throws Exception
{
          File inputFile = createCountInputFile();
          try {
-             pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "';");
+             pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath())
+ "';");
              pigServer.registerQuery("B = group A all;");
              pigServer.registerQuery("C = foreach B generate COUNT(A);");
              assertValidCount();
@@ -908,7 +908,7 @@ public class TestBuiltin {
         File inputFile = Util.createInputFile("tmp", inputFileName, inputData);
 
          try {
-             pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "';");
+             pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath())
+ "';");
              pigServer.registerQuery("B = group A all;");
              pigServer.registerQuery("C = foreach B generate COUNT(A);");
              assertValidCount();
@@ -931,7 +931,7 @@ public class TestBuiltin {
         String inputFileName = file.getAbsolutePath();
 
          try {
-             pigServer.registerQuery("A = load '" + inputFileName + "';");
+             pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFileName) + "';");
              pigServer.registerQuery("B = group A all;");
              pigServer.registerQuery("C = foreach B generate COUNT(A.$0);");
              assertValidCount();
@@ -947,7 +947,7 @@ public class TestBuiltin {
     public void testCount_ValidNumberOfArguments_WithInputSchema_One() throws Exception {
          File inputFile = createCountInputFile();
          try {
-             pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "' as (data:chararray);");
+             pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath())
+ "' as (data:chararray);");
              pigServer.registerQuery("B = group A all;");
              pigServer.registerQuery("C = foreach B generate COUNT(A.$0);");
              assertValidCount();
@@ -962,7 +962,7 @@ public class TestBuiltin {
     public void testCount_ValidNumberOfArguments_WithInputSchema_Two() throws Exception {
          File inputFile = createCountInputFile();
          try {
-             pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "' as (data:chararray);");
+             pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath())
+ "' as (data:chararray);");
              pigServer.registerQuery("B = group A all;");
              pigServer.registerQuery("C = foreach B generate COUNT(A);");
              assertValidCount();
@@ -998,7 +998,7 @@ public class TestBuiltin {
     public void testCount_InvalidNumberOfArguments_WithoutInputSchema() throws Exception
{
          File inputFile = createCountInputFile();
          try {
-             pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "';");
+             pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath())
+ "';");
              pigServer.registerQuery("B = group A all;");
              pigServer.registerQuery("C = foreach B generate COUNT(A.$0, A.$0);");
              pigServer.openIterator("C");
@@ -1014,7 +1014,7 @@ public class TestBuiltin {
     public void testCount_InvalidNumberOfArguments_WithInputSchema() throws Exception {
          File inputFile = createCountInputFile();
          try {
-             pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "' as (data:chararray);");
+             pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath())
+ "' as (data:chararray);");
              pigServer.registerQuery("B = group A all;");
              pigServer.registerQuery("C = foreach B generate COUNT(A.$0, A.$0);");
              pigServer.openIterator("C");
@@ -1030,7 +1030,7 @@ public class TestBuiltin {
     public void testCount_InvalidArgumentType() throws Exception {
          File inputFile = createCountInputFile();
          try {
-             pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "' as (data:chararray);");
+             pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath())
+ "' as (data:chararray);");
              pigServer.registerQuery("B = group A all;");
              pigServer.registerQuery("C = foreach B generate COUNT('data');");
              pigServer.openIterator("C");
@@ -2117,7 +2117,7 @@ public class TestBuiltin {
         File inputFile = Util.createInputFile("tmp", inputFileName, inputData);
 
         // test typed data
-        pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "' AS value:chararray;");
+        pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath())
+ "' AS value:chararray;");
         pigServer.registerQuery("B = foreach A generate STRSPLIT(value, ' ') AS values;");
         pigServer.registerQuery("C = foreach B generate values, SIZE(values) as cnt;");
 
@@ -2264,7 +2264,7 @@ public class TestBuiltin {
         File inputFile = Util.createInputFile("tmp", "testStrUDFsIn.txt", new String[] {inputStr});
 
         // test typed data
-        pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "' as (name:
chararray);");
+        pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath())
+ "' as (name: chararray);");
         pigServer.registerQuery("B = foreach A generate SUBSTRING(name, 0, 3), " +
             "INDEXOF(name, 'a'), INDEXOF(name, 'a', 3), LAST_INDEX_OF(name, 'a'), REPLACE(name,
'a', 'b'), " +
             "STRSPLIT(name), STRSPLIT(name, ' '), STRSPLIT(name, ' ', 0), TRIM(name);");
@@ -2286,7 +2286,7 @@ public class TestBuiltin {
         assertEquals("amy smith", t.get(8));
 
         // test untyped data
-        pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "' as (name);");
+        pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath())
+ "' as (name);");
         pigServer.registerQuery("B = foreach A generate SUBSTRING(name, 0, 3), " +
             "LAST_INDEX_OF(name, 'a'), REPLACE(name, 'a', 'b'), TRIM(name);");
 

Modified: pig/trunk/test/org/apache/pig/test/TestJoin.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestJoin.java?rev=1362673&r1=1362672&r2=1362673&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestJoin.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestJoin.java Tue Jul 17 21:42:38 2012
@@ -92,7 +92,7 @@ public class TestJoin extends TestCase {
             fileName = fileNameHint;
         } else if (execType == ExecType.LOCAL) {
             File f = Util.createInputFile("test", fileNameHint, data);
-            fileName = "file://" + f.getAbsolutePath();
+            fileName = Util.generateURI(Util.encodeEscape(f.getAbsolutePath()), pigServer.getPigContext());
         }
         return fileName;
     }
@@ -173,8 +173,8 @@ public class TestJoin extends TestCase {
             Tuple expectedResult = (Tuple)Util.getPigConstant("('hello',1,'hello','world')");
             
             // with schema
-            String script = "a = load '"+ firstInput +"' as (n:chararray, a:int); " +
-            		"b = load '"+ secondInput +"' as (n:chararray, m:chararray); " +
+            String script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (n:chararray,
a:int); " +
+            		"b = load '"+ Util.encodeEscape(secondInput) +"' as (n:chararray, m:chararray);
" +
             		"c = join a by $0, b by $0;";
             Util.registerMultiLineQuery(pigServer, script);
             Iterator<Tuple> it = pigServer.openIterator("c");
@@ -183,8 +183,8 @@ public class TestJoin extends TestCase {
             assertEquals(false, it.hasNext());
             
             // without schema
-            script = "a = load '"+ firstInput + "'; " +
-            "b = load '" + secondInput + "'; " +
+            script = "a = load '"+ Util.encodeEscape(firstInput) + "'; " +
+            "b = load '" + Util.encodeEscape(secondInput) + "'; " +
             "c = join a by $0, b by $0;";
             Util.registerMultiLineQuery(pigServer, script);
             it = pigServer.openIterator("c");
@@ -216,8 +216,8 @@ public class TestJoin extends TestCase {
             Tuple expectedResult = (Tuple)Util.getPigConstant("(1,2,1,'hello',1,2,1,'hello')");
             
             // with schema
-            String script = "a = load '"+ firstInput +"' as (i:int, j:int); " +
-                    "b = load '"+ secondInput +"' as (k:int, l:chararray); " +
+            String script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (i:int, j:int);
" +
+                    "b = load '"+ Util.encodeEscape(secondInput) +"' as (k:int, l:chararray);
" +
                     "c = join a by $0, b by $0;" +
                     "d = foreach c generate i,j,k,l,a::i as ai,a::j as aj,b::k as bk,b::l
as bl;";
             Util.registerMultiLineQuery(pigServer, script);
@@ -227,8 +227,8 @@ public class TestJoin extends TestCase {
             assertEquals(false, it.hasNext());
             
             // schema with duplicates
-            script = "a = load '"+ firstInput +"' as (i:int, j:int); " +
-            "b = load '"+ secondInput +"' as (i:int, l:chararray); " +
+            script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (i:int, j:int); "
+
+            "b = load '"+ Util.encodeEscape(secondInput) +"' as (i:int, l:chararray); " +
             "c = join a by $0, b by $0;" +
             "d = foreach c generate i,j,l,a::i as ai,a::j as aj,b::i as bi,b::l as bl;";
             boolean exceptionThrown = false;
@@ -243,8 +243,8 @@ public class TestJoin extends TestCase {
             assertEquals(true, exceptionThrown);
             
             // schema with duplicates with resolution
-            script = "a = load '"+ firstInput +"' as (i:int, j:int); " +
-            "b = load '"+ secondInput +"' as (i:int, l:chararray); " +
+            script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (i:int, j:int); "
+
+            "b = load '"+ Util.encodeEscape(secondInput) +"' as (i:int, l:chararray); " +
             "c = join a by $0, b by $0;" +
             "d = foreach c generate a::i as ai1,j,b::i as bi1,l,a::i as ai2,a::j as aj2,b::i
as bi3,b::l as bl3;";
             Util.registerMultiLineQuery(pigServer, script);
@@ -283,11 +283,11 @@ public class TestJoin extends TestCase {
         }
         
         // with schema
-        String script = "a = load '"+ firstInput +"' ; " +
+        String script = "a = load '"+ Util.encodeEscape(firstInput) +"' ; " +
         //re-using alias a for new operator below, doing this intentionally 
         // because such use case has been seen
         "a = foreach a generate $0 as i, $1 as j ;" +
-        "b = load '"+ secondInput +"' as (k, l); " +
+        "b = load '"+ Util.encodeEscape(secondInput) +"' as (k, l); " +
         "c = join a by $0, b by $0;" +
         "d = foreach c generate i,j,k,l,a::i as ai,a::j as aj,b::k as bk,b::l as bl;";
         Util.registerMultiLineQuery(pigServer, script);
@@ -329,8 +329,8 @@ public class TestJoin extends TestCase {
             // with and without optional outer
             for(int i = 0; i < 2; i++) {
                 //with schema
-                String script = "a = load '"+ firstInput +"' as (n:chararray, a:int); " +
-                        "b = load '"+ secondInput +"' as (n:chararray, m:chararray); ";
+                String script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (n:chararray,
a:int); " +
+                        "b = load '"+ Util.encodeEscape(secondInput) +"' as (n:chararray,
m:chararray); ";
                 if(i == 0) {
                     script +=  "c = join a by $0 left outer, b by $0;" ;
                 } else {
@@ -351,8 +351,8 @@ public class TestJoin extends TestCase {
                     assertEquals(expectedResults.size(), counter);
                     
                     // without schema
-                    script = "a = load '"+ firstInput +"'; " +
-                    "b = load '"+ secondInput +"'; ";
+                    script = "a = load '"+ Util.encodeEscape(firstInput) +"'; " +
+                    "b = load '"+ Util.encodeEscape(secondInput) +"'; ";
                     if(i == 0) {
                         script +=  "c = join a by $0 left outer, b by $0;" ;
                     } else {
@@ -398,8 +398,8 @@ public class TestJoin extends TestCase {
             // with and without optional outer
             for(int i = 0; i < 2; i++) {
                 // with schema
-                String script = "a = load '"+ firstInput +"' as (n:chararray, a:int); " +
-                        "b = load '"+ secondInput +"' as (n:chararray, m:chararray); ";
+                String script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (n:chararray,
a:int); " +
+                        "b = load '"+ Util.encodeEscape(secondInput) +"' as (n:chararray,
m:chararray); ";
                 if(i == 0) {
                     script +=  "c = join a by $0 right outer, b by $0;" ;
                 } else {
@@ -420,8 +420,8 @@ public class TestJoin extends TestCase {
                     assertEquals(expectedResults.size(), counter);
                     
                     // without schema
-                    script = "a = load '"+ firstInput +"'; " +
-                    "b = load '"+ secondInput +"'; " ;
+                    script = "a = load '"+ Util.encodeEscape(firstInput) +"'; " +
+                    "b = load '"+ Util.encodeEscape(secondInput) +"'; " ;
                     if(i == 0) {
                         script +=  "c = join a by $0 right outer, b by $0;" ;
                     } else {
@@ -469,8 +469,8 @@ public class TestJoin extends TestCase {
             // with and without optional outer
             for(int i = 0; i < 2; i++) {
                 // with schema
-                String script = "a = load '"+ firstInput +"' as (n:chararray, a:int); " +
-                        "b = load '"+ secondInput +"' as (n:chararray, m:chararray); ";
+                String script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (n:chararray,
a:int); " +
+                        "b = load '"+ Util.encodeEscape(secondInput) +"' as (n:chararray,
m:chararray); ";
                 if(i == 0) {
                     script +=  "c = join a by $0 full outer, b by $0;" ;
                 } else {
@@ -491,8 +491,8 @@ public class TestJoin extends TestCase {
                     assertEquals(expectedResults.size(), counter);
                     
                     // without schema
-                    script = "a = load '"+ firstInput +"'; " +
-                    "b = load '"+ secondInput +"'; " ;
+                    script = "a = load '"+ Util.encodeEscape(firstInput) +"'; " +
+                    "b = load '"+ Util.encodeEscape(secondInput) +"'; " ;
                     if(i == 0) {
                         script +=  "c = join a by $0 full outer, b by $0;" ;
                     } else {
@@ -577,8 +577,8 @@ public class TestJoin extends TestCase {
             String firstInput = createInputFile(execType, "a.txt", input1);
             String secondInput = createInputFile(execType, "b.txt", input2);
             
-            String script = "a = load '"+ firstInput +"' as (a:tuple(a1:int, a2:chararray));"
+
-                    "b = load '"+ secondInput +"' as (b:tuple(b1:int, b2:chararray));" +
+            String script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (a:tuple(a1:int,
a2:chararray));" +
+                    "b = load '"+ Util.encodeEscape(secondInput) +"' as (b:tuple(b1:int,
b2:chararray));" +
                     "c = join a by a.a1, b by b.b1;";
             Util.registerMultiLineQuery(pigServer, script);
             Iterator<Tuple> it = pigServer.openIterator("c");
@@ -612,8 +612,8 @@ public class TestJoin extends TestCase {
             String firstInput = createInputFile(execType, "a.txt", input1);
             String secondInput = createInputFile(execType, "b.txt", input2);
             
-            String script = "a = load '"+ firstInput +"' as (a1:int, a2:chararray);" +
-                    "b = load '"+ secondInput +"' as (b1:int, b2:chararray);" +
+            String script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (a1:int, a2:chararray);"
+
+                    "b = load '"+ Util.encodeEscape(secondInput) +"' as (b1:int, b2:chararray);"
+
                     "c = join a by (a1, a2), b by (b1, b2);";
             Util.registerMultiLineQuery(pigServer, script);
             Iterator<Tuple> it = pigServer.openIterator("c");

Modified: pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java?rev=1362673&r1=1362672&r2=1362673&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java Tue Jul 17 21:42:38 2012
@@ -100,6 +100,7 @@ public class TestPigScriptParser extends
     
     @Test
     public void testDefineUDF() throws Exception {
+        PigServer ps = new PigServer(ExecType.LOCAL);
         String inputData[] = {
                 "dshfdskfwww.xyz.com/sportsjoadfjdslpdshfdskfwww.xyz.com/sportsjoadfjdsl"
,
                 "kas;dka;sd" ,
@@ -108,15 +109,15 @@ public class TestPigScriptParser extends
                 "wwwJxyzMcom/sports"
         };
         File f = Util.createFile(inputData);
+
         String[] queryLines = new String[] { 
                 // the reason we have 4 backslashes below is we really want to put two backslashes
but
                 // since this is to be represented in a Java String, we escape each backslash
with one more
                 // backslash - hence 4. In a pig script in a file, this would be
                 // www\\.xyz\\.com
                 "define minelogs org.apache.pig.test.RegexGroupCount('www\\\\.xyz\\\\.com/sports');"
,
-        		"A = load 'file://" + f.getAbsolutePath() + "'  using PigStorage() as (source :
chararray);" ,
+        		"A = load '" + Util.generateURI(Util.encodeEscape(f.getAbsolutePath()), ps.getPigContext())
+ "'  using PigStorage() as (source : chararray);" ,
         		"B = foreach A generate minelogs(source) as sportslogs;" };
-        PigServer ps = new PigServer(ExecType.LOCAL);
         for (String line : queryLines) {
             ps.registerQuery(line);
         }

Modified: pig/trunk/test/org/apache/pig/test/TestUnion.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestUnion.java?rev=1362673&r1=1362672&r2=1362673&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestUnion.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestUnion.java Tue Jul 17 21:42:38 2012
@@ -96,7 +96,7 @@ public class TestUnion extends junit.fra
         POLoad ld1 = GenPhyOp.topLoadOp();
         String curDir = System.getProperty("user.dir");
         String inpDir = curDir + File.separatorChar + "test/org/apache/pig/test/data/InputFiles/";
-        FileSpec fSpec = new FileSpec("file:"+ inpDir +"passwd", new FuncSpec(PigStorage.class.getName()
, new String[]{":"}));
+        FileSpec fSpec = new FileSpec(Util.generateURI(Util.encodeEscape(inpDir + "passwd"),
pc), new FuncSpec(PigStorage.class.getName() , new String[]{":"}));
         ld1.setLFile(fSpec);
 
         POLoad ld2 = GenPhyOp.topLoadOp();



Mime
View raw message