pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1148118 [3/3] - in /pig/branches/branch-0.9: ./ shims/ shims/src/ shims/src/hadoop20/ shims/src/hadoop20/org/ shims/src/hadoop20/org/apache/ shims/src/hadoop20/org/apache/pig/ shims/src/hadoop20/org/apache/pig/backend/ shims/src/hadoop20/o...
Date Tue, 19 Jul 2011 01:15:02 GMT
Modified: pig/branches/branch-0.9/src/org/apache/pig/impl/io/ReadToEndLoader.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/impl/io/ReadToEndLoader.java?rev=1148118&r1=1148117&r2=1148118&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/impl/io/ReadToEndLoader.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/impl/io/ReadToEndLoader.java Tue Jul 19 01:14:58
2011
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapreduce.TaskA
 import org.apache.pig.LoadCaster;
 import org.apache.pig.LoadFunc;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
+import org.apache.pig.backend.hadoop.executionengine.shims.HadoopShims;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.plan.OperatorKey;
 
@@ -150,7 +151,7 @@ public class ReadToEndLoader extends Loa
         conf = job.getConfiguration();
         inputFormat = wrappedLoadFunc.getInputFormat();
         try {
-            inpSplits = inputFormat.getSplits(new JobContext(conf,
+            inpSplits = inputFormat.getSplits(HadoopShims.createJobContext(conf,
                     new JobID()));
         } catch (InterruptedException e) {
             throw new IOException(e);
@@ -167,7 +168,7 @@ public class ReadToEndLoader extends Loa
             reader.close();
         }
         InputSplit curSplit = inpSplits.get(curSplitIndex);
-        TaskAttemptContext tAContext = new TaskAttemptContext(conf, 
+        TaskAttemptContext tAContext = HadoopShims.createTaskAttemptContext(conf, 
                 new TaskAttemptID());
         reader = inputFormat.createRecordReader(curSplit, tAContext);
         reader.initialize(curSplit, tAContext);

Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserUtils.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserUtils.java?rev=1148118&r1=1148117&r2=1148118&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserUtils.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserUtils.java Tue Jul 19 01:14:58
2011
@@ -97,6 +97,9 @@ public class QueryParserUtils {
     static void setHdfsServers(String absolutePath, PigContext pigContext) throws URISyntaxException
{
         // Get native host
         String defaultFS = (String)pigContext.getProperties().get("fs.default.name");
+        if (defaultFS==null)
+            defaultFS = (String)pigContext.getProperties().get("fs.defaultFS");
+        
         URI defaultFSURI = new URI(defaultFS);
         String defaultHost = defaultFSURI.getHost();
         if (defaultHost == null) defaultHost = "";

Modified: pig/branches/branch-0.9/src/overview.html
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/overview.html?rev=1148118&r1=1148117&r2=1148118&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/overview.html (original)
+++ pig/branches/branch-0.9/src/overview.html Tue Jul 19 01:14:58 2011
@@ -46,9 +46,9 @@ submitted to Hadoop and monitored by the
 {@link org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher}.
 <p>
 On the backend, each 
-{@link org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce.Map},
+{@link org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce.Map},
 {@link org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigCombiner.Combine},
and 
-{@link org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce.Reduce}

+{@link org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce.Reduce}

 use the pipeline of physical operators constructed in the front end to load, process, and
store
 data.
 

Modified: pig/branches/branch-0.9/test/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/findbugsExcludeFile.xml?rev=1148118&r1=1148117&r2=1148118&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/findbugsExcludeFile.xml (original)
+++ pig/branches/branch-0.9/test/findbugsExcludeFile.xml Tue Jul 19 01:14:58 2011
@@ -425,4 +425,8 @@
         <Method name = "mergeCollection" />
         <Bug pattern = "NP_GUARANTEED_DEREF_ON_EXCEPTION_PATH" />
     </Match>
+    <Match>
+        <Class name = "org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce"
/>
+        <Bug pattern = "MS_CANNOT_BE_FINAL" />
+    </Match>
 </FindBugsFilter>

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/MiniCluster.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/MiniCluster.java?rev=1148118&r1=1148117&r2=1148118&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/MiniCluster.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/MiniCluster.java Tue Jul 19 01:14:58
2011
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.pig.test;
-
-import java.io.*;
-import java.util.Properties;
-
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.mapred.MiniMRCluster;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-
-/**
- * This class builds a single instance of itself with the Singleton 
- * design pattern. While building the single instance, it sets up a 
- * mini cluster that actually consists of a mini DFS cluster and a 
- * mini MapReduce cluster on the local machine and also sets up the 
- * environment for Pig to run on top of the mini cluster.
- */
-public class MiniCluster {
-    private MiniDFSCluster m_dfs = null;
-    private MiniMRCluster m_mr = null;
-    private FileSystem m_fileSys = null;
-    private JobConf m_conf = null;
-    
-    private final static MiniCluster INSTANCE = new MiniCluster();
-    private static boolean isSetup = true;
-    
-    private MiniCluster() {
-        setupMiniDfsAndMrClusters();
-    }
-    
-    private void setupMiniDfsAndMrClusters() {
-        try {
-            final int dataNodes = 4;     // There will be 4 data nodes
-            final int taskTrackers = 4;  // There will be 4 task tracker nodes
-            
-            // Create the configuration hadoop-site.xml file
-            File conf_dir = new File(System.getProperty("user.home"), "pigtest/conf/");
-            conf_dir.mkdirs();
-            File conf_file = new File(conf_dir, "hadoop-site.xml");
-            
-            conf_file.delete();
-            
-            // Builds and starts the mini dfs and mapreduce clusters
-            Configuration config = new Configuration();
-            m_dfs = new MiniDFSCluster(config, dataNodes, true, null);
-            m_fileSys = m_dfs.getFileSystem();
-            m_mr = new MiniMRCluster(taskTrackers, m_fileSys.getUri().toString(), 1);
-            
-            // Write the necessary config info to hadoop-site.xml
-            m_conf = m_mr.createJobConf();      
-            m_conf.setInt("mapred.submit.replication", 2);
-            m_conf.set("dfs.datanode.address", "0.0.0.0:0");
-            m_conf.set("dfs.datanode.http.address", "0.0.0.0:0");
-            m_conf.set("mapred.map.max.attempts", "2");
-            m_conf.set("mapred.reduce.max.attempts", "2");
-            m_conf.writeXml(new FileOutputStream(conf_file));
-            
-            // Set the system properties needed by Pig
-            System.setProperty("cluster", m_conf.get("mapred.job.tracker"));
-            System.setProperty("namenode", m_conf.get("fs.default.name"));
-            System.setProperty("junit.hadoop.conf", conf_dir.getPath());
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-    
-    /**
-     * Returns the single instance of class MiniClusterBuilder that
-     * represents the resouces for a mini dfs cluster and a mini 
-     * mapreduce cluster. 
-     */
-    public static MiniCluster buildCluster() {
-        if(! isSetup){
-            INSTANCE.setupMiniDfsAndMrClusters();
-            isSetup = true;
-        }
-        return INSTANCE;
-    }
-
-    public void shutDown(){
-        INSTANCE.shutdownMiniDfsAndMrClusters();
-    }
-    
-    protected void finalize() {
-        shutdownMiniDfsAndMrClusters();
-    }
-    
-    private void shutdownMiniDfsAndMrClusters() {
-        isSetup = false;
-        try {
-            if (m_fileSys != null) { m_fileSys.close(); }
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-        if (m_dfs != null) { m_dfs.shutdown(); }
-        if (m_mr != null) { m_mr.shutdown(); }     
-        m_fileSys = null;
-        m_dfs = null;
-        m_mr = null;
-    }
-
-    public Properties getProperties() {
-        errorIfNotSetup();
-        return ConfigurationUtil.toProperties(m_conf);
-    }
-
-    public Configuration getConfiguration() {
-        return new Configuration(m_conf);
-    }
-
-    public void setProperty(String name, String value) {
-        errorIfNotSetup();
-        m_conf.set(name, value);
-    }
-    
-    public FileSystem getFileSystem() {
-        errorIfNotSetup();
-        return m_fileSys;
-    }
-    
-    /**
-     * Throw RunTimeException if isSetup is false
-     */
-    private void errorIfNotSetup(){
-        if(isSetup)
-            return;
-        String msg = "function called on MiniCluster that has been shutdown";
-        throw new RuntimeException(msg);
-    }
-}

Added: pig/branches/branch-0.9/test/org/apache/pig/test/MiniGenericCluster.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/MiniGenericCluster.java?rev=1148118&view=auto
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/MiniGenericCluster.java (added)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/MiniGenericCluster.java Tue Jul 19 01:14:58
2011
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.io.*;
+import java.util.Properties;
+
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+
+/**
+ * This class builds a single instance of itself with the Singleton 
+ * design pattern. While building the single instance, it sets up a 
+ * mini cluster that actually consists of a mini DFS cluster and a 
+ * mini MapReduce cluster on the local machine and also sets up the 
+ * environment for Pig to run on top of the mini cluster.
+ *
+ * This class is the base class for MiniCluster, which has slightly
+ * difference among different versions of hadoop. MiniCluster implementation
+ * is located in $PIG_HOME/shims.
+ */
+abstract public class MiniGenericCluster {
+    protected MiniDFSCluster m_dfs = null;
+    protected FileSystem m_fileSys = null;
+    protected Configuration m_conf = null;
+    
+    protected final static MiniCluster INSTANCE = new MiniCluster();
+    protected static boolean isSetup = true;
+    
+    protected MiniGenericCluster() {
+        setupMiniDfsAndMrClusters();
+    }
+    
+    abstract protected void setupMiniDfsAndMrClusters();
+    
+    /**
+     * Returns the single instance of class MiniClusterBuilder that
+     * represents the resouces for a mini dfs cluster and a mini 
+     * mapreduce cluster. 
+     */
+    public static MiniCluster buildCluster() {
+        if(! isSetup){
+            INSTANCE.setupMiniDfsAndMrClusters();
+            isSetup = true;
+        }
+        return INSTANCE;
+    }
+
+    public void shutDown(){
+        INSTANCE.shutdownMiniDfsAndMrClusters();
+    }
+    
+    protected void finalize() {
+        shutdownMiniDfsAndMrClusters();
+    }
+    
+    protected void shutdownMiniDfsAndMrClusters() {
+        isSetup = false;
+        shutdownMiniDfsClusters();
+        shutdownMiniMrClusters();
+    }
+    
+    protected void shutdownMiniDfsClusters() {
+        try {
+            if (m_fileSys != null) { m_fileSys.close(); }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        if (m_dfs != null) { m_dfs.shutdown(); }
+        m_fileSys = null;
+        m_dfs = null;
+    }
+    
+    abstract protected void shutdownMiniMrClusters();
+
+    public Properties getProperties() {
+        errorIfNotSetup();
+        return ConfigurationUtil.toProperties(m_conf);
+    }
+
+    public Configuration getConfiguration() {
+        return new Configuration(m_conf);
+    }
+
+    public void setProperty(String name, String value) {
+        errorIfNotSetup();
+        m_conf.set(name, value);
+    }
+    
+    public FileSystem getFileSystem() {
+        errorIfNotSetup();
+        return m_fileSys;
+    }
+    
+    /**
+     * Throw RunTimeException if isSetup is false
+     */
+    private void errorIfNotSetup(){
+        if(isSetup)
+            return;
+        String msg = "function called on MiniCluster that has been shutdown";
+        throw new RuntimeException(msg);
+    }
+}

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestGrunt.java?rev=1148118&r1=1148117&r2=1148118&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestGrunt.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestGrunt.java Tue Jul 19 01:14:58 2011
@@ -522,7 +522,7 @@ public class TestGrunt extends TestCase 
 
     @Test
     public void testIllustrateScript() throws Throwable {
-        PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer server = new PigServer(ExecType.LOCAL, new Properties());
         PigContext context = server.getPigContext();
         
         String strCmd = "illustrate -script "
@@ -538,7 +538,7 @@ public class TestGrunt extends TestCase 
 
     @Test
     public void testIllustrateScript2() throws Throwable {
-        PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer server = new PigServer(ExecType.LOCAL, new Properties());
         PigContext context = server.getPigContext();
         
         String strCmd = "illustrate -script "
@@ -554,7 +554,7 @@ public class TestGrunt extends TestCase 
     
     @Test
     public void testIllustrateScript3() throws Throwable {
-        PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer server = new PigServer(ExecType.LOCAL, new Properties());
         PigContext context = server.getPigContext();
         
         String strCmd = "illustrate -script "
@@ -571,7 +571,7 @@ public class TestGrunt extends TestCase 
     @Test
     public void testIllustrateScript4() throws Throwable {
         // empty line/field test
-        PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer server = new PigServer(ExecType.LOCAL, new Properties());
         PigContext context = server.getPigContext();
         
         String strCmd = "illustrate -script "
@@ -588,7 +588,7 @@ public class TestGrunt extends TestCase 
     @Test
     public void testIllustrateScript5() throws Throwable {
         // empty line/field test
-        PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer server = new PigServer(ExecType.LOCAL, new Properties());
         PigContext context = server.getPigContext();
         
         String strCmd = "illustrate -script "
@@ -605,7 +605,7 @@ public class TestGrunt extends TestCase 
     @Test
     public void testIllustrateScript6() throws Throwable {
         // empty line/field test
-        PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer server = new PigServer(ExecType.LOCAL, new Properties());
         PigContext context = server.getPigContext();
         
         String strCmd = "illustrate -script "
@@ -622,7 +622,7 @@ public class TestGrunt extends TestCase 
     @Test
     public void testIllustrateScript7() throws Throwable {
         // empty line/field test
-        PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer server = new PigServer(ExecType.LOCAL, new Properties());
         PigContext context = server.getPigContext();
         
         String strCmd = "illustrate -script "
@@ -1345,4 +1345,4 @@ public class TestGrunt extends TestCase 
             Util.checkMessageInException(e, errMsg);
         }
     }
-}
\ No newline at end of file
+}

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestTypedMap.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestTypedMap.java?rev=1148118&r1=1148117&r2=1148118&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestTypedMap.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestTypedMap.java Tue Jul 19 01:14:58
2011
@@ -23,6 +23,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Properties;
 
 import junit.framework.Assert;
 
@@ -43,7 +44,6 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 public class TestTypedMap  {
-    static MiniCluster cluster ;
     private String tmpDirName = System.getProperty("java.io.tmpdir") + "/pigtest/tmp";
 
     @Before
@@ -71,19 +71,9 @@ public class TestTypedMap  {
         }
     }
 
-    @BeforeClass
-    public static void oneTimeSetup() throws IOException, Exception {
-        cluster = MiniCluster.buildCluster();
-    }
-
-    @AfterClass
-    public static void oneTimeTearDown() throws Exception {
-        cluster.shutDown();
-    }
-
     @Test
     public void testSimpleLoad() throws IOException, ParserException {
-        PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
         String[] input = {
                 "[key#1,key2#2]",
                 "[key#2]",
@@ -120,7 +110,7 @@ public class TestTypedMap  {
 
     @Test
     public void testSimpleMapKeyLookup() throws IOException, ParserException {
-        PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
         String[] input = {
                 "[key#1,key2#2]",
                 "[key#2]",
@@ -151,7 +141,7 @@ public class TestTypedMap  {
 
     @Test
     public void testSimpleMapCast() throws IOException, ParserException {
-        PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
         String[] input = {
                 "[key#1,key2#2]",
                 "[key#2]",
@@ -189,7 +179,7 @@ public class TestTypedMap  {
 
     @Test
     public void testComplexLoad() throws IOException, ParserException {
-        PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
         String[] input = {
                 "[key#{(1,2),(1,3)},134#]",
                 "[key2#]",
@@ -223,7 +213,7 @@ public class TestTypedMap  {
 
     @Test
     public void testComplexCast() throws IOException, ParserException {
-        PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
         String[] input = {
                 "[key#{(1,2),(1,3)},134#]",
                 "[key2#]",
@@ -258,7 +248,7 @@ public class TestTypedMap  {
 
     @Test
     public void testComplexCast2() throws IOException, ParserException {
-        PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
         String[] input = {
                 "[key#1,key2#2]",
         };
@@ -288,7 +278,7 @@ public class TestTypedMap  {
 
     @Test
     public void testUnTypedMap() throws IOException, ParserException {
-        PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
         String[] input = {
                 "[key#1,key2#2]",
         };
@@ -317,7 +307,7 @@ public class TestTypedMap  {
 
     @Test
     public void testOrderBy() throws IOException, ParserException {
-        PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+        PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
         String[] input = {
                 "[key#1,key1#2]",
                 "[key#2,key3#2]",



Mime
View raw message