pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1752889 - in /pig/branches/branch-0.16: CHANGES.txt src/org/apache/pig/parser/RegisterResolver.java test/org/apache/pig/test/TestRegisterParser.java
Date Sat, 16 Jul 2016 05:06:22 GMT
Author: daijy
Date: Sat Jul 16 05:06:22 2016
New Revision: 1752889

URL: http://svn.apache.org/viewvc?rev=1752889&view=rev
Log:
PIG-4949: Fix registering jar in S3 which was broken by PIG-4417 in Pig 0.16

Modified:
    pig/branches/branch-0.16/CHANGES.txt
    pig/branches/branch-0.16/src/org/apache/pig/parser/RegisterResolver.java
    pig/branches/branch-0.16/test/org/apache/pig/test/TestRegisterParser.java

Modified: pig/branches/branch-0.16/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/CHANGES.txt?rev=1752889&r1=1752888&r2=1752889&view=diff
==============================================================================
--- pig/branches/branch-0.16/CHANGES.txt (original)
+++ pig/branches/branch-0.16/CHANGES.txt Sat Jul 16 05:06:22 2016
@@ -133,6 +133,8 @@ PIG-4639: Add better parser for Apache H
 
 BUG FIXES
 
+PIG-4949: Fix registering jar in S3 which was broken by PIG-4417 in Pig 0.16 (yangyishan0901m
via daijy)
+
 PIG-4821: Pig chararray field with special UTF-8 chars as part of tuple join key produces
wrong results in Tez (rohini)
 
 PIG-4734: TOMAP schema inferring breaks some scripts in type checking for bincond (daijy)

Modified: pig/branches/branch-0.16/src/org/apache/pig/parser/RegisterResolver.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/src/org/apache/pig/parser/RegisterResolver.java?rev=1752889&r1=1752888&r2=1752889&view=diff
==============================================================================
--- pig/branches/branch-0.16/src/org/apache/pig/parser/RegisterResolver.java (original)
+++ pig/branches/branch-0.16/src/org/apache/pig/parser/RegisterResolver.java Sat Jul 16 05:06:22
2016
@@ -23,6 +23,10 @@ import java.net.URISyntaxException;
 
 import org.apache.pig.PigServer;
 import org.apache.pig.tools.DownloadResolver;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+import org.apache.pig.backend.hadoop.executionengine.shims.HadoopShims;
+import org.apache.hadoop.fs.Path;
 
 public class RegisterResolver {
 
@@ -66,15 +70,24 @@ public class RegisterResolver {
         String scheme = uri.getScheme();
         if (scheme != null) {
             scheme = scheme.toLowerCase();
+            if (scheme.equals("ivy")) {
+                DownloadResolver downloadResolver = DownloadResolver.getInstance();
+                return downloadResolver.downloadArtifact(uri, pigServer);
+            }
+            if (!hasFileSystemImpl(uri)) {
+                throw new ParserException("Invalid Scheme: " + uri.getScheme());
+            }
         }
-        if (scheme == null || scheme.equals("file") || scheme.equals("hdfs")) {
-            return new URI[] { uri };
-        } else if (scheme.equals("ivy")) {
-            DownloadResolver downloadResolver = DownloadResolver.getInstance();
-            return downloadResolver.downloadArtifact(uri, pigServer);
-        } else {
-            throw new ParserException("Invalid Scheme: " + uri.getScheme());
-        }
+        return new URI[] { uri };
+    }
+   
+    /**
+     * @param uri
+     * @return True if the uri has valid file system implementation
+     */ 
+    private boolean hasFileSystemImpl(URI uri) {
+      Configuration conf = ConfigurationUtil.toConfiguration(pigServer.getPigContext().getProperties(),
true);
+      return HadoopShims.hasFileSystemImpl(new Path(uri), conf);
     }
 
     /**

Modified: pig/branches/branch-0.16/test/org/apache/pig/test/TestRegisterParser.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/test/org/apache/pig/test/TestRegisterParser.java?rev=1752889&r1=1752888&r2=1752889&view=diff
==============================================================================
--- pig/branches/branch-0.16/test/org/apache/pig/test/TestRegisterParser.java (original)
+++ pig/branches/branch-0.16/test/org/apache/pig/test/TestRegisterParser.java Sat Jul 16 05:06:22
2016
@@ -30,6 +30,7 @@ import java.io.IOException;
 import java.io.Writer;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.Properties;
 
 import org.apache.pig.ExecType;
 import org.apache.pig.impl.PigContext;
@@ -42,6 +43,9 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mockito;
+import org.apache.pig.impl.util.PropertiesUtil;
+import org.apache.hadoop.fs.LocalFileSystem;
+
 
 public class TestRegisterParser {
     private PigServer pigServer;
@@ -49,7 +53,12 @@ public class TestRegisterParser {
 
     @Before
     public void setUp() throws Exception {
-	pigServer = new PigServer(ExecType.LOCAL);
+        Properties properties = PropertiesUtil.loadDefaultProperties();
+        properties.setProperty("fs.s3.impl", LocalFileSystem.class.getName());
+        properties.setProperty("fs.s3n.impl", LocalFileSystem.class.getName());
+        properties.setProperty("fs.s3a.impl", LocalFileSystem.class.getName());
+
+	pigServer = new PigServer(ExecType.LOCAL, properties);
 
 	// Generate test jar files
 	for (int i = 1; i <= 5; i++) {
@@ -107,6 +116,34 @@ public class TestRegisterParser {
 	}
     }
 
+    @Test
+    public void testResolveForVariousFileSystemSchemes() throws URISyntaxException, IOException,
ParserException {
+        URI[] list = new URI[6];
+        list[0] = new URI("file://test.jar");
+        list[1] = new URI("hdfs://test.jar");
+        list[2] = new URI("s3://test.jar");
+        list[3] = new URI("s3n://test.jar");
+        list[4] = new URI("s3a://test.jar");
+        list[5] = new URI("test.jar");
+
+        RegisterResolver registerResolver = new RegisterResolver(pigServer);
+        for (URI uri : list) {
+            URI[] resolvedUris = registerResolver.resolve(uri);
+	    Assert.assertEquals(1, resolvedUris.length);
+            Assert.assertEquals(uri, resolvedUris[0]);
+        }
+    }
+
+    @Test(expected = ParserException.class)
+    public void testResolveParseException() throws URISyntaxException, IOException, ParserException
{
+        new RegisterResolver(pigServer).resolve(new URI("abc://test.jar"));
+    }
+
+    @Test(expected = URISyntaxException.class)
+    public void testResolveURISyntaxException() throws URISyntaxException, IOException, ParserException
{
+        new RegisterResolver(pigServer).resolve(new URI("123://test.jar"));
+    }
+
     // Throw error when a scripting language and namespace is specified for a jar
     @Test(expected = ParserException.class)
     public void testRegisterJarException1() throws IOException, ParserException {



Mime
View raw message