pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dvrya...@apache.org
Subject svn commit: r1376815 - in /pig/trunk: ./ src/org/apache/pig/impl/io/ src/org/apache/pig/parser/ test/org/apache/pig/test/
Date Fri, 24 Aug 2012 02:57:06 GMT
Author: dvryaboy
Date: Fri Aug 24 02:57:05 2012
New Revision: 1376815

URL: http://svn.apache.org/viewvc?rev=1376815&view=rev
Log:
PIG-2850: Pig should support loading macro files as resources stored in JAR files

Added:
    pig/trunk/src/org/apache/pig/impl/io/ResourceNotFoundException.java
Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java
    pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
    pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1376815&r1=1376814&r2=1376815&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Aug 24 02:57:05 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
 
 IMPROVEMENTS
 
+PIG-2850: Pig should support loading macro files as resources stored in JAR files (matterhayes
via dvryaboy)
+
 PIG-1314: Add DateTime Support to Pig (zjshen via thejas)
 
 PIG-2785: NoClassDefFoundError after upgrading to pig 0.10.0 from 0.9.0 (matterhayes via
sms)

Modified: pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java?rev=1376815&r1=1376814&r2=1376815&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java (original)
+++ pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java Fri Aug 24 02:57:05 2012
@@ -17,6 +17,7 @@
  */
 package org.apache.pig.impl.io;
 
+import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -818,4 +819,36 @@ public class FileLocalizer {
 
         return fetchFiles;
     }
+    
+    /**
+     * Ensures that the passed resource is available from the local file system, fetching
+     * it to a temporary directory.
+     * 
+     * @throws ResourceNotFoundException 
+     */
+    public static FetchFileRet fetchResource(String name) throws IOException, ResourceNotFoundException
{
+      FetchFileRet localFileRet = null;
+      InputStream resourceStream = PigContext.getClassLoader().getResourceAsStream(name);
+      if (resourceStream != null) {        
+        File dest = new File(localTempDir, name);
+        dest.getParentFile().mkdirs();        
+        dest.deleteOnExit();
+                
+        OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(dest));
+        byte[] buffer = new byte[1024];
+        int len;
+        while ((len=resourceStream.read(buffer)) > 0) {
+          outputStream.write(buffer,0,len);
+        }
+        outputStream.close();
+        
+        localFileRet = new FetchFileRet(dest,false);
+      }
+      else
+      {
+        throw new ResourceNotFoundException(name);
+      }
+      
+      return localFileRet;
+    }
 }

Added: pig/trunk/src/org/apache/pig/impl/io/ResourceNotFoundException.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/io/ResourceNotFoundException.java?rev=1376815&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/io/ResourceNotFoundException.java (added)
+++ pig/trunk/src/org/apache/pig/impl/io/ResourceNotFoundException.java Fri Aug 24 02:57:05
2012
@@ -0,0 +1,18 @@
+package org.apache.pig.impl.io;
+
+public class ResourceNotFoundException extends Exception
+{
+  private static final long serialVersionUID = 1L;
+
+  private String resourceName;
+  
+  public ResourceNotFoundException(String resourceName)
+  {
+    this.resourceName = resourceName;
+  }
+  
+  public String toString()
+  {
+    return "Resource '" + resourceName + "' does not exist";
+  }
+}

Modified: pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java?rev=1376815&r1=1376814&r2=1376815&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java Fri Aug 24 02:57:05 2012
@@ -22,7 +22,10 @@ import java.io.BufferedReader;
 import java.io.FileReader;
 import java.io.File;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -39,9 +42,13 @@ import org.antlr.runtime.tree.CommonTree
 import org.antlr.runtime.tree.Tree;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.pig.PigException;
+import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.io.FileLocalizer;
 import org.apache.pig.impl.io.FileLocalizer.FetchFileRet;
+import org.apache.pig.impl.io.ResourceNotFoundException;
 import org.apache.pig.newplan.Operator;
 import org.apache.pig.newplan.logical.relational.LogicalPlan;
 import org.apache.pig.newplan.logical.relational.LogicalSchema;
@@ -349,11 +356,34 @@ public class QueryParserDriver {
             if (fnameMap.get(fname) != null) {
                 localFileRet = fnameMap.get(fname);
             } else {
+              try {
                 File localFile = QueryParserUtils.getFileFromImportSearchPath(fname);
                 localFileRet = localFile == null ?
                         FileLocalizer.fetchFile(pigContext.getProperties(), fname)
                         : new FetchFileRet(localFile.getCanonicalFile(), false);
-                fnameMap.put(fname, localFileRet);
+              } catch (FileNotFoundException e) {
+                // ignore this since we'll attempt to load as a resource before failing
+                LOG.debug(String.format("Macro file %s was not found", fname));
+              }
+              
+              // try loading the macro file as a resource in case it is packaged in a registered
jar
+              if (localFileRet == null) {
+                LOG.debug(String.format("Attempting to load macro file %s as a resource",
fname));
+                
+                try
+                {
+                  localFileRet = FileLocalizer.fetchResource(fname);
+                  LOG.debug(String.format("Found macro file %s as resource", fname));
+                }
+                catch (ResourceNotFoundException e)
+                {
+                  LOG.debug(String.format("Macro file %s was not found as resource either",
fname));
+                  LOG.error(String.format("Failed to find macro file %s", fname));
+                  throw new ExecException("file '" + fname + "' does not exist.", 101, PigException.INPUT);
+                }
+              }
+                            
+              fnameMap.put(fname, localFileRet);
             }
         } catch (IOException e) {
             throw new RuntimeException("Unable to fetch macro file '" + fname + "'", e);
@@ -369,11 +399,13 @@ public class QueryParserDriver {
     private PigMacro makeMacroDef(CommonTree t, Map<String, PigMacro> seen)
             throws ParserException {
         String mn = t.getChild(0).getText();
- 
+        
         if (!macroSeen.add(mn)) {
             String msg = getErrorMessage(null, t, null,
                     "Duplicated macro name '" + mn + "'");
-            throw new ParserException(msg);
+            LOG.warn(msg);
+            // Disabling this exception, see https://issues.apache.org/jira/browse/PIG-2279
+            // throw new ParserException(msg);
         }
 
         if (seen != null) {
@@ -383,7 +415,7 @@ public class QueryParserDriver {
         }
         
         String fname = ((PigParserNode)t).getFileName();
-
+        
         Tree defNode = t.getChild(1);
 
         // get parameter markers
@@ -446,7 +478,9 @@ public class QueryParserDriver {
         if (!importSeen.add(fname)) {
             String msg = getErrorMessage(fname, t,
                     ": Duplicated import file '" + fname + "'", null);
-            throw new ParserException(msg);
+            LOG.warn(msg);
+            // Disabling this exception, see https://issues.apache.org/jira/browse/PIG-2279
+            // throw new ParserException(msg);
         }
         
         FetchFileRet localFileRet = getMacroFile(fname);

Modified: pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java?rev=1376815&r1=1376814&r2=1376815&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java Fri Aug 24 02:57:05 2012
@@ -30,16 +30,21 @@ import org.junit.Before;
 import org.junit.Test;
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.FileWriter;
 import java.io.PrintWriter;
 import java.util.Iterator;
+import java.util.jar.Attributes;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
 
 import static org.apache.pig.builtin.mock.Storage.resetData;
 import static org.apache.pig.builtin.mock.Storage.tuple;
 
 public class TestPigServerWithMacros {
     private PigServer pig = null;
-
+    
     @Before
     public void setUp() throws Exception{
         pig = new PigServer(ExecType.LOCAL);
@@ -120,4 +125,40 @@ public class TestPigServerWithMacros {
         Assert.assertTrue(iter.next().get(0).equals("Hello, World"));
         Assert.assertFalse(iter.hasNext());
     }
+    
+    @Test
+    public void testRegisterResourceMacro() throws Throwable {
+        String macrosFile = "test/pig/macros.pig";
+        File macrosJarFile = File.createTempFile("macros", ".jar");
+        
+        System.out.println("Creating macros jar " + macrosJarFile);
+        
+        Manifest manifest = new Manifest();
+        manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+        
+        JarOutputStream jarStream = new JarOutputStream(new FileOutputStream(macrosJarFile),
manifest);
+        
+        JarEntry jarEntry = new JarEntry(macrosFile);
+        jarEntry.setTime(System.currentTimeMillis());
+        jarStream.putNextEntry(jarEntry);        
+        
+        PrintWriter pw = new PrintWriter(jarStream);
+        pw.println("DEFINE row_count_in_jar(X) RETURNS Z { Y = group $X all; $Z = foreach
Y generate COUNT($X); };");
+        pw.close();        
+        
+        jarStream.close();
+        
+        Storage.Data data = resetData(pig);
+        data.set("some_path", "(l:int)", tuple(tuple("1")), tuple(tuple("2")), tuple(tuple("3")),
tuple(tuple("10")), tuple(tuple("11")));
+                
+        System.out.println("Registering macros jar " + macrosJarFile);
+        pig.registerJar(macrosJarFile.toString());
+        
+        pig.registerQuery("import '" + macrosFile + "';");
+        pig.registerQuery("a = load 'some_path' USING mock.Storage();");
+        pig.registerQuery("b = row_count_in_jar(a);");
+        Iterator<Tuple> iter = pig.openIterator("b");
+        
+        Assert.assertTrue(((Long)iter.next().get(0))==5);
+    }
 }



Mime
View raw message