incubator-bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject svn commit: r1199014 - in /incubator/bigtop/branches/hadoop-0.23: bigtop-packages/src/common/sqoop/patch bigtop-packages/src/deb/sqoop/rules bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec bigtop.mk
Date Mon, 07 Nov 2011 23:39:42 GMT
Author: rvs
Date: Mon Nov  7 23:39:41 2011
New Revision: 1199014

URL: http://svn.apache.org/viewvc?rev=1199014&view=rev
Log:
Making sqoop builds against .23

Modified:
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/sqoop/patch
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/sqoop/rules
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec
    incubator/bigtop/branches/hadoop-0.23/bigtop.mk

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/sqoop/patch
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/sqoop/patch?rev=1199014&r1=1199013&r2=1199014&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/sqoop/patch (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/sqoop/patch Mon Nov 
7 23:39:41 2011
@@ -1,43 +1,252 @@
-Index: sqoop-1.3.0/ivy.xml
-===================================================================
---- sqoop-1.3.0.orig/ivy.xml	2011-09-27 10:28:59.474837063 -0700
-+++ sqoop-1.3.0/ivy.xml	2011-09-27 10:28:46.733590903 -0700
-@@ -66,10 +66,18 @@
+diff --git ivy.xml ivy.xml
+index d686c76..a882dae 100644
+--- ivy.xml
++++ ivy.xml
+@@ -67,12 +67,19 @@ under the License.
+     <artifact conf="master"/>
    </publications>
    <dependencies>
-     <!-- Dependencies for Cloudera's Distribution for Hadoop -->
--    <dependency org="com.cloudera.hadoop" name="hadoop-core"
+-    <!-- Dependencies for Cloudera's Distribution for Hadoop -->
+-    <dependency org="org.apache.hadoop" name="hadoop-core"
 -      rev="${hadoop-core.cloudera.version}" conf="cloudera->default"/>
--    <dependency org="com.cloudera.hadoop" name="hadoop-test"
+-    <dependency org="org.apache.hadoop" name="hadoop-test"
 -      rev="${hadoop-core.cloudera.version}" conf="clouderatest->default"/>
-+    <dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop-common.version}"
+-
++    <dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop.version}"
 +      conf="cloudera->default"/> 
-+    <dependency org="org.apache.hadoop" name="hadoop-common-test" rev="${hadoop-common.version}"
-+      conf="clouderatest->default"/> 
-+    <dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="${hadoop-hdfs.version}"
++    <dependency org="org.apache.hadoop" name="hadoop-common-test" rev="${hadoop.version}"
++                conf="clouderatest->default"/> 
++    <dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="${hadoop.version}"
 +      conf="cloudera->default"/>
-+    <dependency org="org.apache.hadoop" name="hadoop-hdfs-test" rev="${hadoop-hdfs.version}"
++    <dependency org="org.apache.hadoop" name="hadoop-hdfs-test" rev="${hadoop.version}"
 +      conf="clouderatest->default"/>
-+    <dependency org="org.apache.hadoop" name="hadoop-mapred" rev="${hadoop-mapred.version}"
++    <dependency org="org.apache.hadoop" name="hadoop-mapred" rev="${hadoop.version}"
 +       conf="cloudera->default"/>
-+    <dependency org="org.apache.hadoop" name="hadoop-mapred-test" rev="${hadoop-mapred.version}"
++    <dependency org="org.apache.hadoop" name="hadoop-mapred-test" rev="${hadoop.version}"
 +      conf="clouderatest->default"/>
-
- 
++    
      <!-- Common dependencies for Sqoop -->
      <dependency org="commons-cli" name="commons-cli"
-Index: sqoop-1.3.0/ivy/libraries.properties
-===================================================================
---- sqoop-1.3.0.orig/ivy/libraries.properties	2011-09-27 10:30:10.136087155 -0700
-+++ sqoop-1.3.0/ivy/libraries.properties	2011-09-27 10:29:29.734839438 -0700
-@@ -27,7 +27,9 @@
+       rev="${commons-cli.version}" conf="common->default"/>
+@@ -121,9 +128,13 @@ under the License.
+       <exclude org="com.sun.jersey" module="jersey-server"/>
+       <exclude org="org.apache.thrift" module="thrift"/>
+       <exclude org="log4j" module="log4j"/>
++      <exclude org="org.apache.hadoop" module="hadoop-core" />
++      <exclude org="com.cloudera.cdh" module="zookeeper-ant" />
+     </dependency>
+ 
+     <exclude org="org.apache.hadoop" module="avro"/>
++    <exclude org="commons-configuration" module="commons-configuration" />
++    <exclude org="commons-daemon" module="commons-daemon" />
+ 
+   </dependencies>
+ </ivy-module>
+diff --git ivy/libraries.properties b/ivy/libraries.properties
+index dbbcb3b..8ba9a8b 100644
+--- ivy/libraries.properties
++++ ivy/libraries.properties
+@@ -29,14 +29,14 @@ commons-lang.version=2.4
  commons-logging.version=1.0.4
  
  # Cloudera Distribution dependency version
--hadoop-core.cloudera.version=0.20.2-737
-+hadoop-common.version=0.23.0-SNAPSHOT
-+hadoop-hdfs.version=0.23.0-SNAPSHOT
-+hadoop-mapred.version=0.23.0-SNAPSHOT
+-hadoop-core.cloudera.version=0.20.2-cdh3u1
++hadoop.version=0.23.0-SNAPSHOT
+ 
+ hbase.version=0.90.3-cdh3u1
+ zookeeper.version=3.3.3-cdh3u1
+ 
+ hsqldb.version=1.8.0.10
+ 
+-ivy.version=2.0.0-rc2
++ivy.version=2.1.0
+ 
+ junit.version=4.5
+ 
+Index: src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java
+===================================================================
+--- src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java	(revision 1198954)
++++ src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java	(working copy)
+@@ -31,10 +31,7 @@
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.fs.FileSystem;
+ import org.apache.hadoop.fs.Path;
+-import org.apache.hadoop.mapreduce.MapContext;
+-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+ 
+-import com.cloudera.sqoop.testutil.MockObjectFactory;
+ import com.cloudera.sqoop.testutil.MockResultSet;
+ 
+ /**
+@@ -45,7 +42,6 @@
+   protected Configuration conf;
+   protected LargeObjectLoader loader;
+   protected Path outDir;
+-  protected MapContext mapContext;
+ 
+   public void setUp() throws IOException, InterruptedException {
+     conf = new Configuration();
+@@ -60,17 +56,7 @@
+     }
+     fs.mkdirs(outDir);
+ 
+-    /* A mock MapContext that uses FileOutputCommitter.
+-     * This MapContext is actually serving two roles here; when writing the
+-     * CLOB files, its OutputCommitter is used to determine where to write
+-     * the CLOB data, as these are placed in the task output work directory.
+-     * When reading the CLOB data back for verification, we use the
+-     * getInputSplit() to determine where to read our source data from--the same
+-     * directory. We are repurposing the same context for both output and input.
+-     */
+-    mapContext = MockObjectFactory.getMapContextForIOPath(conf, outDir);
+-    loader = new LargeObjectLoader(mapContext.getConfiguration(),
+-        FileOutputFormat.getWorkOutputPath(mapContext));
++    loader = new LargeObjectLoader(conf, outDir);
+   }
+ 
+   public void testReadClobRef()
+@@ -88,7 +74,6 @@
+     assertNotNull(clob);
+     assertTrue(clob.isExternal());
+     loader.close();
+-    mapContext.getOutputCommitter().commitTask(mapContext);
+     Reader r = clob.getDataStream(conf, outDir);
+     char [] buf = new char[4096];
+     int chars = r.read(buf, 0, 4096);
+@@ -117,7 +102,6 @@
+     assertNotNull(blob);
+     assertTrue(blob.isExternal());
+     loader.close();
+-    mapContext.getOutputCommitter().commitTask(mapContext);
+     InputStream is = blob.getDataStream(conf, outDir);
+     byte [] buf = new byte[4096];
+     int bytes = is.read(buf, 0, 4096);
+Index: src/test/com/cloudera/sqoop/testutil/MockObjectFactory.java
+===================================================================
+--- src/test/com/cloudera/sqoop/testutil/MockObjectFactory.java	(revision 1198954)
++++ src/test/com/cloudera/sqoop/testutil/MockObjectFactory.java	(working copy)
+@@ -1,84 +0,0 @@
+-/**
+- * Licensed to the Apache Software Foundation (ASF) under one
+- * or more contributor license agreements.  See the NOTICE file
+- * distributed with this work for additional information
+- * regarding copyright ownership.  The ASF licenses this file
+- * to you under the Apache License, Version 2.0 (the
+- * "License"); you may not use this file except in compliance
+- * with the License.  You may obtain a copy of the License at
+- *
+- *     http://www.apache.org/licenses/LICENSE-2.0
+- *
+- * Unless required by applicable law or agreed to in writing, software
+- * distributed under the License is distributed on an "AS IS" BASIS,
+- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+- * See the License for the specific language governing permissions and
+- * limitations under the License.
+- */
+-
+-package com.cloudera.sqoop.testutil;
+-
+-import java.io.IOException;
+-
+-import org.apache.hadoop.conf.Configuration;
+-import org.apache.hadoop.fs.Path;
+-import org.apache.hadoop.mapreduce.Counters;
+-import org.apache.hadoop.mapreduce.InputSplit;
+-import org.apache.hadoop.mapreduce.MapContext;
+-import org.apache.hadoop.mapreduce.OutputCommitter;
+-import org.apache.hadoop.mapreduce.TaskAttemptID;
+-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+-import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
+-import org.apache.hadoop.mrunit.mapreduce.mock.MockReporter;
+-
+-/**
+- * Allows the creation of various mock objects for testing purposes.
+- */
+-public final class MockObjectFactory {
+-
+-  /**
+-   * Returns a mock MapContext that has both an OutputCommitter and an
+-   * InputSplit wired to the specified path.
+-   * Used for testing LargeObjectLoader.
+-   */
+-  public static MapContext getMapContextForIOPath(Configuration conf, Path p) {
+-    return new MockMapContextWithCommitter(conf, p);
+-  }
+-
+-  private static class MockMapContextWithCommitter
+-      extends MapContext<Object, Object, Object, Object> {
+-    private Path path;
+-    private Configuration conf;
+-
+-    public MockMapContextWithCommitter(Configuration c, Path p) {
+-      super(c, new TaskAttemptID("jt", 0, true, 0, 0),
+-            null, null, null, new MockReporter(new Counters()), null);
+-
+-      this.path = p;
+-      this.conf = c;
+-    }
+-
+-    @Override
+-    public OutputCommitter getOutputCommitter() {
+-      try {
+-        return new FileOutputCommitter(path, this);
+-      } catch (IOException ioe) {
+-        return null;
+-      }
+-    }
+-
+-    @Override
+-    public InputSplit getInputSplit() {
+-      return new FileSplit(new Path(path, "inputFile"), 0, 0, new String[0]);
+-    }
+-
+-    @Override
+-    public Configuration getConfiguration() {
+-      return conf;
+-    }
+-  }
+-
+-  private MockObjectFactory() {
+-    // Disable explicity object creation
+-  }
+-}
+Index: src/test/com/cloudera/sqoop/mapreduce/db/TestDataDrivenDBInputFormat.java
+===================================================================
+--- src/test/com/cloudera/sqoop/mapreduce/db/TestDataDrivenDBInputFormat.java	(revision 1198954)
++++ src/test/com/cloudera/sqoop/mapreduce/db/TestDataDrivenDBInputFormat.java	(working copy)
+@@ -23,13 +23,14 @@
+ import java.io.DataOutput;
+ import java.io.IOException;
+ 
++import junit.framework.TestCase;
++
+ import org.apache.commons.logging.Log;
+ import org.apache.commons.logging.LogFactory;
+ import org.apache.hadoop.fs.*;
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.io.NullWritable;
+ import org.apache.hadoop.io.WritableComparable;
+-import org.apache.hadoop.mapred.HadoopTestCase;
+ import org.apache.hadoop.mapreduce.*;
+ import org.apache.hadoop.mapreduce.lib.db.*;
+ import org.apache.hadoop.mapreduce.lib.output.*;
+@@ -38,7 +39,7 @@
+ /**
+  * Test aspects of DataDrivenDBInputFormat.
+  */
+-public class TestDataDrivenDBInputFormat extends HadoopTestCase {
++public class TestDataDrivenDBInputFormat extends TestCase {
+ 
+   private static final Log LOG = LogFactory.getLog(
+       TestDataDrivenDBInputFormat.class);
+@@ -52,10 +53,6 @@
+ 
+   private static final String OUT_DIR;
  
- hbase.version=0.89.20100924-28
- zookeeper.version=3.3.1+7
+-  public TestDataDrivenDBInputFormat() throws IOException {
+-    super(LOCAL_MR, LOCAL_FS, 1, 1);
+-  }
+-
+   static {
+     OUT_DIR = System.getProperty("test.build.data", "/tmp") + "/dddbifout";
+   }

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/sqoop/rules
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/sqoop/rules?rev=1199014&r1=1199013&r2=1199014&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/sqoop/rules (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/sqoop/rules Mon Nov  7 23:39:41
2011
@@ -25,7 +25,7 @@ export DH_OPTIONS
 
 patch: patch-stamp
 patch-stamp:	
-	patch -p1 < debian/patch
+	patch -p0 < debian/patch
 	touch $@
 
 clean:	

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec?rev=1199014&r1=1199013&r2=1199014&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec Mon
Nov  7 23:39:41 2011
@@ -79,8 +79,8 @@ Shared metadata repository for Sqoop. Th
 server for Sqoop clients across a network to use.
 
 %prep
-%setup -n sqoop-%{sqoop_base_version}
-%patch0 -p1
+%setup -n apache-sqoop-e9d0be
+%patch0 -p0
 
 %build
 bash %{SOURCE1} -Dversion=%{version}

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop.mk
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop.mk?rev=1199014&r1=1199013&r2=1199014&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop.mk (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop.mk Mon Nov  7 23:39:41 2011
@@ -86,12 +86,14 @@ $(eval $(call PACKAGE,hive,HIVE))
 SQOOP_NAME=sqoop
 SQOOP_RELNOTES_NAME=Sqoop
 SQOOP_PKG_NAME=sqoop
-SQOOP_BASE_VERSION=1.3.0
-SQOOP_PKG_VERSION=1.3.0
+SQOOP_BASE_VERSION=1.4.0
+SQOOP_PKG_VERSION=1.4.0
 SQOOP_RELEASE_VERSION=1
 SQOOP_TARBALL_DST=sqoop-$(SQOOP_BASE_VERSION).tar.gz
-SQOOP_TARBALL_SRC=$(SQOOP_TARBALL_DST)
-SQOOP_SITE=http://github.com/downloads/cloudera/sqoop/
+#SQOOP_TARBALL_SRC=$(SQOOP_TARBALL_DST)
+#SQOOP_SITE=http://github.com/downloads/cloudera/sqoop/
+SQOOP_SITE=https://github.com/apache/sqoop/tarball
+SQOOP_TARBALL_SRC=e9d0bed
 $(eval $(call PACKAGE,sqoop,SQOOP))
 
 # Oozie



Mime
View raw message