incubator-bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject svn commit: r1214001 - /incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch
Date Wed, 14 Dec 2011 00:27:53 GMT
Author: rvs
Date: Wed Dec 14 00:27:53 2011
New Revision: 1214001

URL: http://svn.apache.org/viewvc?rev=1214001&view=rev
Log:
Updating PIG's patch

Modified:
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch?rev=1214001&r1=1214000&r2=1214001&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch Wed Dec 14
00:27:53 2011
@@ -1,98 +1,185 @@
-Index: shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
-===================================================================
---- shims/test/hadoop23/org/apache/pig/test/MiniCluster.java	(revision 1208121)
-+++ shims/test/hadoop23/org/apache/pig/test/MiniCluster.java	(working copy)
-@@ -92,14 +92,16 @@
- 				e.printStackTrace();
- 			}
- 			*/
--			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"),
new Path("/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"));
--			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"),
new Path("/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"));
--			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///pig.jar"), new Path("/pig.jar"));
--			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///pig-test.jar"), new Path("/pig-test.jar"));
-+			File jarBaseDir = new File("./build/ivy/lib/Pig");
-+			
-+			m_dfs.getFileSystem().copyFromLocalFile(new Path(new File(jarBaseDir, "hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar").toURI()),
new Path("/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"));
-+			m_dfs.getFileSystem().copyFromLocalFile(new Path(new File(jarBaseDir, "hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar").toURI()),
new Path("/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"));
-+			m_dfs.getFileSystem().copyFromLocalFile(new Path(new File("pig-withouthadoop.jar").toURI()),
new Path("/pig-withouthadoop.jar"));
-+			//m_dfs.getFileSystem().copyFromLocalFile(new Path(new File(jarBaseDir, "pig-test.jar").toURI()),
new Path("/pig-test.jar"));
+diff --git build.xml build.xml
+index 1edc448..e8af7ce 100644
+--- build.xml
++++ build.xml
+@@ -156,7 +156,7 @@
+            input="${hadoop-core.version}"
+            regexp="\d+\.(\d+)\.\d+"
+            select="\1" /-->
+-    <property name="hadoopversion" value="20" />
++    <property name="hadoopversion" value="23" />
  
-             DistributedCache.addFileToClassPath(new Path("/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"),
m_conf);
--            DistributedCache.addFileToClassPath(new Path("/pig.jar"), m_conf);
--            DistributedCache.addFileToClassPath(new Path("/pig-test.jar"), m_conf);
-+            DistributedCache.addFileToClassPath(new Path("/pig-withouthadoop.jar"), m_conf);
-+            //DistributedCache.addFileToClassPath(new Path("/pig-test.jar"), m_conf);
-             DistributedCache.addFileToClassPath(new Path("/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"),
m_conf);
-             String cachefile = m_conf.get("mapreduce.job.cache.files");
-             m_conf.set("alternative.mapreduce.job.cache.files", cachefile);
-Index: ivy.xml
-===================================================================
---- ivy.xml	2011-11-21 17:26:11.000000000 -0800
-+++ ivy.xml	2011-12-02 20:49:31.149818867 -0800
-@@ -67,41 +67,50 @@
-     <dependency org="org.apache.hadoop" name="hadoop-test" rev="${hadoop-test.version}"
-       conf="hadoop20->default"/>
-     <dependency org="org.apache.hadoop" name="hadoop-annotations" 
--      rev="${hadoop-common.version}" conf="hadoop23->master"/>
-+      rev="${hadoop-common.version}" conf="hadoop23->default"/>
-     <dependency org="org.apache.hadoop" name="hadoop-auth" 
--      rev="${hadoop-common.version}" conf="hadoop23->master"/>
-+      rev="${hadoop-common.version}" conf="hadoop23->default"/>
-     <dependency org="org.apache.hadoop" name="hadoop-common" 
--      rev="${hadoop-common.version}" conf="hadoop23->master">
-+      rev="${hadoop-common.version}" conf="hadoop23->default">
-       <artifact name="hadoop-common" ext="jar" />
-       <artifact name="hadoop-common" type="tests" ext="jar" m:classifier="tests" />
-     </dependency>
-     <dependency org="org.apache.hadoop" name="hadoop-hdfs"
--      rev="${hadoop-hdfs.version}" conf="hadoop23->master">
-+      rev="${hadoop-hdfs.version}" conf="hadoop23->default">
-       <artifact name="hadoop-hdfs" ext="jar" />
-       <artifact name="hadoop-hdfs" type="tests" ext="jar" m:classifier="tests" />
-+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
-+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
-     </dependency>
-+    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-app" rev="${hadoop-mapreduce.version}"
-+      conf="hadoop23->default"/>
-+    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-common" 
-+      rev="${hadoop-mapreduce.version}" conf="hadoop23->default"/>
-     <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core" rev="${hadoop-mapreduce.version}"
--      conf="hadoop23->master"/>
-+      conf="hadoop23->default"/>
-+    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-hs" rev="${hadoop-mapreduce.version}"
-+      conf="hadoop23->default"/>
+     <condition property="isHadoop23">
+         <equals arg1="hadoopversion" arg2="23"/>
+@@ -415,6 +415,10 @@
+         </antcall>
+     	
+     	<copy file="${basedir}/test/hbase-site.xml" tofile="${test.build.classes}/hbase-site.xml"/>
++   
++        <ivy:cachepath pathid="mr-apps-test.classpath" />
++        <property name="mr-apps-classpath" refid="mr-apps-test.classpath" />
++        <echo file="${test.build.classes}/mrapp-generated-classpath" message="${mr-apps-classpath}"
/>
+     </target>
+ 
+     <!-- This target is for default compilation -->
+diff --git ivy.xml ivy.xml
+index 9fc2f94..85803c5 100644
+--- ivy.xml
++++ ivy.xml
+@@ -60,6 +60,24 @@
+       conf="hadoop23->master"/>
+     <dependency org="commons-configuration" name="commons-configuration" rev="${commons-configuration.version}"
+       conf="hadoop23->master"/>
++    <dependency org="commons-collections" name="commons-collections" rev="${commons-collections.version}"
++      conf="hadoop23->master"/>
++    <dependency org="javax.servlet" name="servlet-api" rev="${servlet-api.version}"
++      conf="hadoop23->master"/>
++    <dependency org="javax.ws.rs" name="jsr311-api" rev="${jsr311-api.version}"
++      conf="hadoop23->master"/>
++    <dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}"
++      conf="hadoop23->master"/>
++    <dependency org="com.google.protobuf" name="protobuf-java" rev="${protobuf-java.version}"
++      conf="hadoop23->master"/>
++    <dependency org="org.mortbay.jetty" name="jetty-util" rev="${jetty-util.version}"
++      conf="hadoop23->master"/>
++    <dependency org="com.google.inject" name="guice" rev="${guice.version}"
++      conf="hadoop23->master"/>
++    <dependency org="com.google.inject.extensions" name="guice-servlet" rev="${guice-servlet.version}"
++      conf="hadoop23->master"/>
++    <dependency org="aopalliance" name="aopalliance" rev="${aopalliance.version}"
++      conf="hadoop23->master"/>
+     <dependency org="log4j" name="log4j" rev="${log4j.version}"
+       conf="compile->master"/>
+     <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop-core.version}"
+@@ -84,7 +102,8 @@
+       conf="hadoop23->master"/>
      <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-jobclient" rev="${hadoop-mapreduce.version}"
--      conf="hadoop23->master">
-+      conf="hadoop23->default">
-+        <artifact name="hadoop-mapreduce-client-jobclient" type="jar"/>
-         <artifact name="hadoop-mapreduce-client-jobclient" type="jar" m:classifier="tests"/>
+       conf="hadoop23->master">
+-        <artifact name="hadoop-mapreduce-client-jobclient" type="jar" m:classifier="tests"/>
++        <artifact name="hadoop-mapreduce-client-jobclient" ext="jar" />
++        <artifact name="hadoop-mapreduce-client-jobclient" type="tests" ext="jar" m:classifier="tests"/>
          <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
          <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
      </dependency>
--    <dependency org="org.apache.hadoop" name="hadoop-yarn-server-tests" rev="${hadoop-mapreduce.version}"
--      conf="hadoop23->master">
--      <artifact name="hadoop-yarn-server-tests" type="jar" m:classifier="tests"/>
--    </dependency>
--    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-common"
--      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
-+    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-shuffle" 
-+      rev="${hadoop-mapreduce.version}" conf="hadoop23->default"/>
+@@ -92,16 +111,28 @@
+       conf="hadoop23->master">
+       <artifact name="hadoop-yarn-server-tests" type="jar" m:classifier="tests"/>
+     </dependency>
++    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-app" rev="${hadoop-mapreduce.version}"
++      conf="hadoop23->master" />
++    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-shuffle" rev="${hadoop-mapreduce.version}"
++      conf="hadoop23->master" />
+     <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-common"
+       rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
      <dependency org="org.apache.hadoop" name="hadoop-yarn-api" 
 -      rev="${hadoop-common.version}" conf="hadoop23->master"/>
-+      rev="${hadoop-common.version}" conf="hadoop23->default"/>
++      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
      <dependency org="org.apache.hadoop" name="hadoop-yarn-common" 
 -      rev="${hadoop-common.version}" conf="hadoop23->master"/>
-+      rev="${hadoop-common.version}" conf="hadoop23->default"/>
++      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
++    <dependency org="org.apache.hadoop" name="hadoop-yarn-server" 
++      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
++    <dependency org="org.apache.hadoop" name="hadoop-yarn-server-web-proxy" 
++      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
++    <dependency org="org.apache.hadoop" name="hadoop-yarn-server-common" 
++      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
      <dependency org="org.apache.hadoop" name="hadoop-yarn-server-nodemanager" 
 -      rev="${hadoop-common.version}" conf="hadoop23->master"/>
-+      rev="${hadoop-common.version}" conf="hadoop23->default"/>
++      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
      <dependency org="org.apache.hadoop" name="hadoop-yarn-server-resourcemanager" 
 -      rev="${hadoop-common.version}" conf="hadoop23->master"/>
-+      rev="${hadoop-common.version}" conf="hadoop23->default"/>
-+    <dependency org="org.apache.hadoop" name="hadoop-yarn-server-tests" rev="${hadoop-mapreduce.version}"
-+      conf="hadoop23->master">
-+      <artifact name="hadoop-yarn-server-tests" type="jar" m:classifier="tests"/>
-+    </dependency>
++      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
++    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-hs" 
++      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
      <dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}"
        conf="compile->master"/>
      <dependency org="org.mortbay.jetty" name="jetty-util" rev="${jetty-util.version}"
+diff --git ivy/libraries.properties ivy/libraries.properties
+index 5af5c3a..f0deea9 100644
+--- ivy/libraries.properties
++++ ivy/libraries.properties
+@@ -24,14 +24,15 @@ commons-logging.version=1.1.1
+ commons-lang.version=2.4
+ commons-configuration.version=1.6
+ commons-httpclient.version=3.1
++commons-collections.version=3.2.1
+ checkstyle.version=4.2
+ ivy.version=2.2.0
+ guava.version=r06
+ hadoop-core.version=0.20.2
+ hadoop-test.version=0.20.2
+-hadoop-common.version=0.23.0-SNAPSHOT
+-hadoop-hdfs.version=0.23.0-SNAPSHOT
+-hadoop-mapreduce.version=0.23.0-SNAPSHOT
++hadoop-common.version=0.23.1-SNAPSHOT
++hadoop-hdfs.version=0.23.1-SNAPSHOT
++hadoop-mapreduce.version=0.23.1-SNAPSHOT
+ hbase.version=0.90.0
+ hsqldb.version=1.8.0.10
+ jackson.version=1.7.3
+@@ -57,3 +58,10 @@ slf4j-log4j12.version=1.6.1
+ xerces.version=1.4.4
+ wagon-http.version=1.0-beta-2
+ zookeeper.version=3.3.3
++servlet.version=4.0.6
++servlet-api.version=2.5
++protobuf-java.version=2.4.0a
++guice.version=2.0
++guice-servlet.version=2.0
++aopalliance.version=1.0
++jsr311-api.version=1.1.1
+diff --git shims/test/hadoop23/org/apache/pig/test/MiniCluster.java shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
+index 61f6372..8c3f468 100644
+--- shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
++++ shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
+@@ -72,7 +72,7 @@ public class MiniCluster extends MiniGenericCluster {
+             m_dfs_conf = m_dfs.getConfiguration(0);
+             
+             m_mr = new MiniMRYarnCluster("PigMiniCluster");
+-            m_mr.init(new Configuration());
++            m_mr.init(config);
+             //m_mr.init(m_dfs_conf);
+             m_mr.start();
+             
+@@ -82,27 +82,6 @@ public class MiniCluster extends MiniGenericCluster {
+             
+             m_conf = m_mr_conf;
+ 			m_conf.set("fs.default.name", m_dfs_conf.get("fs.default.name"));
+-
+-			/*
+-			try {
+-				DistributedCache.addCacheFile(new URI("file:///hadoop-mapreduce-client-app-1.0-SNAPSHOT.jar"),
m_conf);
+-				DistributedCache.addCacheFile(new URI("file:///hadoop-mapreduce-client-jobclient-1.0-SNAPSHOT.jar"),
m_conf);
+-				DistributedCache.addCacheFile(new URI("file:///pig.jar"), m_conf);
+-			} catch (Exception e) {
+-				e.printStackTrace();
+-			}
+-			*/
+-			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"),
new Path("/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"));
+-			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"),
new Path("/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"));
+-			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///pig.jar"), new Path("/pig.jar"));
+-			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///pig-test.jar"), new Path("/pig-test.jar"));
+-
+-            DistributedCache.addFileToClassPath(new Path("/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"),
m_conf);
+-            DistributedCache.addFileToClassPath(new Path("/pig.jar"), m_conf);
+-            DistributedCache.addFileToClassPath(new Path("/pig-test.jar"), m_conf);
+-            DistributedCache.addFileToClassPath(new Path("/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"),
m_conf);
+-            String cachefile = m_conf.get("mapreduce.job.cache.files");
+-            m_conf.set("alternative.mapreduce.job.cache.files", cachefile);
+             m_conf.unset("mapreduce.job.cache.files");
+ 
+             //ConfigurationUtil.mergeConf(m_conf, m_dfs_conf);
+diff --git src/org/apache/pig/tools/pigstats/PigStatsUtil.java src/org/apache/pig/tools/pigstats/PigStatsUtil.java
+index 289159c..13eafa5 100644
+--- src/org/apache/pig/tools/pigstats/PigStatsUtil.java
++++ src/org/apache/pig/tools/pigstats/PigStatsUtil.java
+@@ -51,7 +51,8 @@ public abstract class PigStatsUtil {
+     public static final String TASK_COUNTER_GROUP 
+             = "org.apache.hadoop.mapred.Task$Counter";
+     public static final String FS_COUNTER_GROUP 
+-            = "FileSystemCounters";
++            = "org.apache.hadoop.mapreduce.FileSystemCounter";
++            //= "FileSystemCounters";
+     public static final String MAP_INPUT_RECORDS 
+             = "MAP_INPUT_RECORDS";
+     public static final String MAP_OUTPUT_RECORDS 



Mime
View raw message