pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1753026 - in /pig/branches/branch-0.16: CHANGES.txt src/org/apache/pig/Main.java src/org/apache/pig/tools/pigstats/PigStats.java test/org/apache/pig/test/TestPigRunner.java test/org/apache/pig/test/Util.java
Date Sun, 17 Jul 2016 08:25:57 GMT
Author: daijy
Date: Sun Jul 17 08:25:57 2016
New Revision: 1753026

URL: http://svn.apache.org/viewvc?rev=1753026&view=rev
Log:
PIG-4950: Fix minor issues with running scripts in non-local FileSystems

Modified:
    pig/branches/branch-0.16/CHANGES.txt
    pig/branches/branch-0.16/src/org/apache/pig/Main.java
    pig/branches/branch-0.16/src/org/apache/pig/tools/pigstats/PigStats.java
    pig/branches/branch-0.16/test/org/apache/pig/test/TestPigRunner.java
    pig/branches/branch-0.16/test/org/apache/pig/test/Util.java

Modified: pig/branches/branch-0.16/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/CHANGES.txt?rev=1753026&r1=1753025&r2=1753026&view=diff
==============================================================================
--- pig/branches/branch-0.16/CHANGES.txt (original)
+++ pig/branches/branch-0.16/CHANGES.txt Sun Jul 17 08:25:57 2016
@@ -32,6 +32,8 @@ BUG FIXES
 
 PIG-4949: Fix registering jar in S3 which was broken by PIG-4417 in Pig 0.16 (yangyishan0901m
via daijy)
 
+PIG-4950: Fix minor issues with running scripts in non-local FileSystems (petersla via daijy)
+
 Release 0.16.0
 
 INCOMPATIBLE CHANGES

Modified: pig/branches/branch-0.16/src/org/apache/pig/Main.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/src/org/apache/pig/Main.java?rev=1753026&r1=1753025&r2=1753026&view=diff
==============================================================================
--- pig/branches/branch-0.16/src/org/apache/pig/Main.java (original)
+++ pig/branches/branch-0.16/src/org/apache/pig/Main.java Sun Jul 17 08:25:57 2016
@@ -477,7 +477,7 @@ public class Main {
                 }
 
 
-                logFileName = validateLogFile(logFileName, file);
+                logFileName = validateLogFile(logFileName, localFileRet.file);
                 pigContext.getProperties().setProperty("pig.logfile", (logFileName == null?
"": logFileName));
 
                 // Set job name based on name of the script
@@ -488,7 +488,7 @@ public class Main {
                     new File(substFile).deleteOnExit();
                 }
 
-                scriptState.setScript(new File(file));
+                scriptState.setScript(localFileRet.file);
 
                 grunt = new Grunt(pin, pigContext);
                 gruntCalled = true;
@@ -605,7 +605,7 @@ public class Main {
                     return ReturnCode.SUCCESS;
                 }
 
-                logFileName = validateLogFile(logFileName, remainders[0]);
+                logFileName = validateLogFile(logFileName, localFileRet.file);
                 pigContext.getProperties().setProperty("pig.logfile", (logFileName == null?
"": logFileName));
 
                 if (!debug) {
@@ -971,11 +971,10 @@ public class Main {
             System.out.println("Additionally, any Hadoop property can be specified.");
     }
 
-    private static String validateLogFile(String logFileName, String scriptName) {
+    private static String validateLogFile(String logFileName, File scriptFile) {
         String strippedDownScriptName = null;
 
-        if(scriptName != null) {
-            File scriptFile = new File(scriptName);
+        if (scriptFile != null) {
             if(!scriptFile.isDirectory()) {
                 String scriptFileAbsPath;
                 try {

Modified: pig/branches/branch-0.16/src/org/apache/pig/tools/pigstats/PigStats.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/src/org/apache/pig/tools/pigstats/PigStats.java?rev=1753026&r1=1753025&r2=1753026&view=diff
==============================================================================
--- pig/branches/branch-0.16/src/org/apache/pig/tools/pigstats/PigStats.java (original)
+++ pig/branches/branch-0.16/src/org/apache/pig/tools/pigstats/PigStats.java Sun Jul 17 08:25:57
2016
@@ -265,6 +265,13 @@ public abstract class PigStats {
         return ScriptState.get().getPigVersion();
     }
 
+    /**
+     *  Returns the contents of the script that was run.
+     */
+    public String getScript() {
+        return ScriptState.get().getScript();
+    }
+
     public String getScriptId() {
         return ScriptState.get().getId();
     }

Modified: pig/branches/branch-0.16/test/org/apache/pig/test/TestPigRunner.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/test/org/apache/pig/test/TestPigRunner.java?rev=1753026&r1=1753025&r2=1753026&view=diff
==============================================================================
--- pig/branches/branch-0.16/test/org/apache/pig/test/TestPigRunner.java (original)
+++ pig/branches/branch-0.16/test/org/apache/pig/test/TestPigRunner.java Sun Jul 17 08:25:57
2016
@@ -19,6 +19,7 @@ package org.apache.pig.test;
 
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
@@ -35,6 +36,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.Counters;
@@ -59,7 +62,9 @@ import org.junit.AfterClass;
 import org.junit.Assume;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 
 public class TestPigRunner {
 
@@ -70,6 +75,9 @@ public class TestPigRunner {
     private static final String OUTPUT_FILE = "output";
     private static final String PIG_FILE = "test.pig";
 
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
     @BeforeClass
     public static void setUpBeforeClass() throws Exception {
         cluster = MiniGenericCluster.buildCluster();
@@ -1110,6 +1118,77 @@ public class TestPigRunner {
             Util.deleteFile(cluster, "tmp/output");
         }
     }
+
+    @Test
+    public void testStoredScriptContents() throws Exception {
+        String scriptContents = "sh echo success;\n";
+        FileUtils.writeStringToFile(new File(PIG_FILE), scriptContents);
+        Util.copyFromLocalToCluster(cluster, PIG_FILE, PIG_FILE);
+
+        Path inputInDfs = new Path(cluster.getFileSystem().getHomeDirectory(), PIG_FILE);
+        try {
+            runAndValidateStoredScriptContents(PIG_FILE, scriptContents);
+            runAndValidateStoredScriptContents(inputInDfs.toString(), scriptContents);
+        } finally {
+            FileUtils.deleteQuietly(new File(PIG_FILE));
+            Util.deleteQuietly(cluster, PIG_FILE);
+        }
+    }
+
+    @Test
+    public void testErrorLogUnderCustomDir() throws Exception {
+        try (PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE))) {
+            w.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
+            w.println("B = foreach A generate StringSize(a0);");
+            w.println("store B into '" + OUTPUT_FILE + "';");
+        }
+        Util.copyFromLocalToCluster(cluster, PIG_FILE, PIG_FILE);
+
+        Path inputInDfs = new Path(cluster.getFileSystem().getHomeDirectory(), PIG_FILE);
+        try {
+            runAndValidateCustomErrorLogDir(PIG_FILE);
+            runAndValidateCustomErrorLogDir(inputInDfs.toString());
+        } finally {
+            FileUtils.deleteQuietly(new File(PIG_FILE));
+            Util.deleteQuietly(cluster, PIG_FILE);
+        }
+    }
+
+    private void runAndValidateStoredScriptContents(String scriptPath, String expectedContents)
{
+        PigStats stats = runPigLocally(scriptPath);
+        assertTrue(stats.isSuccessful());
+        assertEquals(expectedContents, stats.getScript());
+
+        stats = runPigLocally("-f", scriptPath);
+        assertTrue(stats.isSuccessful());
+        assertEquals(expectedContents, stats.getScript());
+    }
+
+    private void runAndValidateCustomErrorLogDir(String scriptPath) throws IOException {
+        File logsFolder = temporaryFolder.newFolder();
+        String logsPath = logsFolder.getAbsolutePath();
+        assertFileCountUnderDir(logsFolder, 0);
+
+        PigStats stats = runPigLocally("-l", logsPath, scriptPath);
+        assertFalse(stats.isSuccessful());
+        assertFileCountUnderDir(logsFolder, 1);
+
+        stats = runPigLocally("-l", logsPath, "-f", scriptPath);
+        assertFalse(stats.isSuccessful());
+        assertFileCountUnderDir(logsFolder, 2);
+    }
+
+    private void assertFileCountUnderDir(File directory, int expectedFileCount) throws IOException
{
+        String[] files = directory.list();
+        assertNotNull(files);
+        assertEquals(expectedFileCount, files.length);
+    }
+
+    private PigStats runPigLocally(String... extraArgs) {
+        String[] args = ArrayUtils.addAll(new String[]{"-x", "local"}, extraArgs);
+        return PigRunner.run(args, new TestNotificationListener("local"));
+    }
+
     public static class TestNotificationListener implements PigProgressNotificationListener
{
 
         private Map<String, int[]> numMap = new HashMap<String, int[]>();

Modified: pig/branches/branch-0.16/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/test/org/apache/pig/test/Util.java?rev=1753026&r1=1753025&r2=1753026&view=diff
==============================================================================
--- pig/branches/branch-0.16/test/org/apache/pig/test/Util.java (original)
+++ pig/branches/branch-0.16/test/org/apache/pig/test/Util.java Sun Jul 17 08:25:57 2016
@@ -480,6 +480,19 @@ public class Util {
         fs.delete(new Path(fileName), true);
     }
 
+    /**
+     * Deletes a dfs file from the MiniCluster DFS quietly
+     *
+     * @param miniCluster the MiniCluster where the file should be deleted
+     * @param fileName the path of the file to be deleted
+     */
+     public static void deleteQuietly(MiniGenericCluster miniCluster, String fileName) {
+         try {
+             deleteFile(miniCluster, fileName);
+         } catch (IOException ignored) {
+         }
+     }
+
     static public void deleteFile(PigContext pigContext, String fileName)
     throws IOException {
         Configuration conf = ConfigurationUtil.toConfiguration(



Mime
View raw message