kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject [2/6] kudu git commit: [kudu-spark] Allow build against Spark 2.0 with Scala 2.11
Date Wed, 02 Nov 2016 21:39:44 GMT
[kudu-spark] Allow build against Spark 2.0 with Scala 2.11

This patch adds build profiles to build an artifact for Spark 2.0
with Scala 2.11. There was one tiny code change to package.scala
required to support Spark 2.0. To handle this, the different
versions of the package.scala reside in different directories and
the build profile includes the proper one.

The two artifactIds are
- kudu-spark_2.10
- kudu-spark2_2.11

The Jenkins job builds and tests both the default and spark2_2.11
profiles.

Change-Id: Ie055ac1e8f7be7cb992124afa0b601376db14f91
Reviewed-on: http://gerrit.cloudera.org:8080/4886
Tested-by: Kudu Jenkins
Reviewed-by: Jean-Daniel Cryans <jdcryans@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e46e9518
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e46e9518
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e46e9518

Branch: refs/heads/master
Commit: e46e9518b12471efdd65e3a989e05d4ffbbc96c9
Parents: 3a3f714
Author: Will Berkeley <wdberkeley@gmail.com>
Authored: Sun Oct 30 18:48:38 2016 -0400
Committer: Will Berkeley <wdberkeley@gmail.com>
Committed: Wed Nov 2 18:15:54 2016 +0000

----------------------------------------------------------------------
 build-support/jenkins/build-and-test.sh         |  7 ++++
 java/README.md                                  | 14 ++++++++
 java/kudu-spark/pom.xml                         | 33 ++++++++++++++++-
 .../org/apache/kudu/spark/kudu/package.scala    | 38 --------------------
 .../org/apache/kudu/spark/kudu/package.scala    | 38 ++++++++++++++++++++
 .../org/apache/kudu/spark/kudu/package.scala    | 38 ++++++++++++++++++++
 6 files changed, 129 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/e46e9518/build-support/jenkins/build-and-test.sh
----------------------------------------------------------------------
diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh
index ca27217..6448e17 100755
--- a/build-support/jenkins/build-and-test.sh
+++ b/build-support/jenkins/build-and-test.sh
@@ -350,6 +350,13 @@ if [ "$BUILD_JAVA" == "1" ]; then
     EXIT_STATUS=1
     FAILURES="$FAILURES"$'Java build/test failed\n'
   fi
+  # Test kudu-spark with Spark 2.x + Scala 2.11 profile
+  # This won't work if there are ever Spark integration tests!
+  rm -rf kudu-spark/target/
+  if ! mvn $MVN_FLAGS -Pspark2_2.11 -Dtest="org.apache.kudu.spark.kudu.*" test; then
+    EXIT_STATUS=1
+    FAILURES="$FAILURES"$'Java build/test failed\n'
+  fi
   set +x
   popd
 fi

http://git-wip-us.apache.org/repos/asf/kudu/blob/e46e9518/java/README.md
----------------------------------------------------------------------
diff --git a/java/README.md b/java/README.md
index 345e030..ac45186 100644
--- a/java/README.md
+++ b/java/README.md
@@ -51,6 +51,20 @@ may be run with:
 
 $ mvn verify
 
+Building the Kudu-Spark integration for Spark 2.x with Scala 2.11
+------------------------------------------------------------
+
+The Spark integration builds for Spark 1.x and Scala 2.10 by default.
+Additionally, there is a build profile available for Spark 2.x with
+Scala 2.11: from the kudu-spark directory, run
+
+$ mvn clean package -P spark2_2.11
+
+The two artifactIds are
+
+1. kudu-spark_2.10 for Spark 1.x with Scala 2.10
+2. kudu-spark2_2.11 for Spark 2.x with Scala 2.11
+
 State of Eclipse integration
 ------------------------------------------------------------
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/e46e9518/java/kudu-spark/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-spark/pom.xml b/java/kudu-spark/pom.xml
index 4c4cda9..f9a0fcc 100644
--- a/java/kudu-spark/pom.xml
+++ b/java/kudu-spark/pom.xml
@@ -20,14 +20,16 @@
         <version>1.1.0-SNAPSHOT</version>
     </parent>
 
-    <artifactId>kudu-spark_${scala.binary.version}</artifactId>
+    <artifactId>kudu-${spark.version.label}_${scala.binary.version}</artifactId>
     <name>Kudu Spark Bindings</name>
 
     <properties>
         <spark.version>1.6.1</spark.version>
+        <spark.version.label>spark</spark.version.label>
         <scala.version>2.10.4</scala.version>
         <scala.binary.version>2.10</scala.binary.version>
         <top.dir>${project.basedir}/..</top.dir>
+        <compat.src>src/main/spark1</compat.src>
     </properties>
 
     <dependencies>
@@ -161,6 +163,35 @@
                     </execution>
                 </executions>
             </plugin>
+            <!-- http://stackoverflow.com/questions/270445 -->
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>generate-sources</phase>
+                        <goals><goal>add-source</goal></goals>
+                        <configuration>
+                            <sources>
+                                <source>${compat.src}</source>
+                            </sources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
+
+    <profiles>
+        <profile>
+            <id>spark2_2.11</id>
+            <properties>
+                <spark.version>2.0.1</spark.version>
+                <compat.src>src/main/spark2</compat.src>
+                <spark.version.label>spark2</spark.version.label>
+                <scala.version>2.11.6</scala.version>
+                <scala.binary.version>2.11</scala.binary.version>
+            </properties>
+        </profile>
+    </profiles>
 </project>

http://git-wip-us.apache.org/repos/asf/kudu/blob/e46e9518/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/package.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/package.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/package.scala
deleted file mode 100755
index 6f49023..0000000
--- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/package.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kudu.spark
-
-import org.apache.spark.sql.{DataFrame, DataFrameReader, DataFrameWriter}
-
-package object kudu {
-
-  /**
-   * Adds a method, `kudu`, to DataFrameReader that allows you to read Kudu tables using
-   * the DataFrameReader.
-   */
-  implicit class KuduDataFrameReader(reader: DataFrameReader) {
-    def kudu: DataFrame = reader.format("org.apache.kudu.spark.kudu").load
-  }
-
-  /**
-    * Adds a method, `kudu`, to DataFrameWriter that allows writes to Kudu using
-    * the DataFileWriter
-    */
-  implicit class KuduDataFrameWriter(writer: DataFrameWriter) {
-    def kudu = writer.format("org.apache.kudu.spark.kudu").save
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kudu/blob/e46e9518/java/kudu-spark/src/main/spark1/org/apache/kudu/spark/kudu/package.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/spark1/org/apache/kudu/spark/kudu/package.scala b/java/kudu-spark/src/main/spark1/org/apache/kudu/spark/kudu/package.scala
new file mode 100755
index 0000000..6f49023
--- /dev/null
+++ b/java/kudu-spark/src/main/spark1/org/apache/kudu/spark/kudu/package.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kudu.spark
+
+import org.apache.spark.sql.{DataFrame, DataFrameReader, DataFrameWriter}
+
+package object kudu {
+
+  /**
+   * Adds a method, `kudu`, to DataFrameReader that allows you to read Kudu tables using
+   * the DataFrameReader.
+   */
+  implicit class KuduDataFrameReader(reader: DataFrameReader) {
+    def kudu: DataFrame = reader.format("org.apache.kudu.spark.kudu").load
+  }
+
+  /**
+    * Adds a method, `kudu`, to DataFrameWriter that allows writes to Kudu using
+    * the DataFileWriter
+    */
+  implicit class KuduDataFrameWriter(writer: DataFrameWriter) {
+    def kudu = writer.format("org.apache.kudu.spark.kudu").save
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kudu/blob/e46e9518/java/kudu-spark/src/main/spark2/org/apache/kudu/spark/kudu/package.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/spark2/org/apache/kudu/spark/kudu/package.scala b/java/kudu-spark/src/main/spark2/org/apache/kudu/spark/kudu/package.scala
new file mode 100755
index 0000000..19ae9ef
--- /dev/null
+++ b/java/kudu-spark/src/main/spark2/org/apache/kudu/spark/kudu/package.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kudu.spark
+
+import org.apache.spark.sql.{DataFrame, DataFrameReader, DataFrameWriter}
+
+package object kudu {
+
+  /**
+   * Adds a method, `kudu`, to DataFrameReader that allows you to read Kudu tables using
+   * the DataFrameReader.
+   */
+  implicit class KuduDataFrameReader(reader: DataFrameReader) {
+    def kudu: DataFrame = reader.format("org.apache.kudu.spark.kudu").load
+  }
+
+  /**
+    * Adds a method, `kudu`, to DataFrameWriter that allows writes to Kudu using
+    * the DataFileWriter
+    */
+  implicit class KuduDataFrameWriter[T](writer: DataFrameWriter[T]) {
+    def kudu = writer.format("org.apache.kudu.spark.kudu").save
+  }
+}
\ No newline at end of file


Mime
View raw message