kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From granthe...@apache.org
Subject [1/2] kudu git commit: [Java] Fix the use of resource files in tests
Date Thu, 12 Jul 2018 19:20:45 GMT
Repository: kudu
Updated Branches:
  refs/heads/master 978cf3912 -> f62bbdd64


[Java] Fix the use of resource files in tests

This is preliminary work needed to run the java tests
on dist-test.

Our tests currently reference files/resources that are
not available in the output test artifacts. This means
our tests work when they are run from source in the
expected location but fail when they are run from the
compiled artifacts.

This fixes those failing tests by discovering the correct
location via Class.getResource and ensuring all needed
files are packaged in the resources directory.

Change-Id: Ia3a90bac67652b58f2eef802c509983c68d8a299
Reviewed-on: http://gerrit.cloudera.org:8080/10924
Reviewed-by: Andrew Wong <awong@cloudera.com>
Tested-by: Grant Henke <granthenke@apache.org>
Reviewed-by: Mike Percy <mpercy@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fff3cf09
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fff3cf09
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fff3cf09

Branch: refs/heads/master
Commit: fff3cf0923782cab8a43e081caeccc88a53f11f9
Parents: 978cf39
Author: Grant Henke <granthenke@apache.org>
Authored: Wed Jul 11 14:17:04 2018 -0500
Committer: Grant Henke <granthenke@apache.org>
Committed: Thu Jul 12 17:14:53 2018 +0000

----------------------------------------------------------------------
 java/kudu-flume-sink/build.gradle               |  6 ++++++
 java/kudu-flume-sink/pom.xml                    |  3 +++
 .../avro/testAvroKuduOperationsProducer.avsc    | 14 -------------
 .../sink/AvroKuduOperationsProducerTest.java    | 22 ++++++++++----------
 .../testAvroKuduOperationsProducer.avsc         | 14 +++++++++++++
 .../spark/tools/TestImportExportFiles.scala     | 10 +++++----
 6 files changed, 40 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/build.gradle
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/build.gradle b/java/kudu-flume-sink/build.gradle
index 5767103..e09f0ae 100644
--- a/java/kudu-flume-sink/build.gradle
+++ b/java/kudu-flume-sink/build.gradle
@@ -33,4 +33,10 @@ dependencies {
 
   testCompile project(path: ":kudu-client", configuration: "shadowTest")
   testCompile libs.junit
+}
+
+// Configure the Avro plugin to compile the schemas in the
+// resource directory so the schemas are available in tests.
+generateTestAvroJava {
+  source("src/test/resources")
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/pom.xml b/java/kudu-flume-sink/pom.xml
index aa8a859..7b664fb 100644
--- a/java/kudu-flume-sink/pom.xml
+++ b/java/kudu-flume-sink/pom.xml
@@ -126,6 +126,9 @@
                 <goal>schema</goal>
               </goals>
               <configuration>
+                <!-- Configure the Avro plugin to compile the schemas in the
+                     resource directory so the schemas are available in tests. -->
+                <testSourceDirectory>${basedir}/src/test/resources</testSourceDirectory>
                 <enableDecimalLogicalType>true</enableDecimalLogicalType>
               </configuration>
           </execution>

http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc b/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc
deleted file mode 100644
index 6bcf6d2..0000000
--- a/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc
+++ /dev/null
@@ -1,14 +0,0 @@
-{"namespace": "org.apache.kudu.flume.sink",
-  "type": "record",
-  "name": "AvroKuduOperationsProducerTestRecord",
-  "fields": [
-    {"name": "key", "type": "int"},
-    {"name": "longField",  "type": "long"},
-    {"name": "doubleField",  "type": "double"},
-    {"name": "nullableField",  "type": ["string", "null"]},
-    {"name": "stringField", "type": "string"},
-    {"name": "decimalField", "type": {
-        "type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 1}
-    }
-  ]
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java
b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java
index 8e3c8dc..42c94cd 100644
--- a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java
+++ b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java
@@ -33,8 +33,9 @@ import static org.junit.Assert.assertEquals;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.IOException;
 import java.math.BigDecimal;
+import java.net.URL;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -51,15 +52,12 @@ import org.apache.avro.specific.SpecificDatumWriter;
 import org.apache.flume.Channel;
 import org.apache.flume.Context;
 import org.apache.flume.Event;
-import org.apache.flume.FlumeException;
 import org.apache.flume.Sink;
 import org.apache.flume.Transaction;
 import org.apache.flume.channel.MemoryChannel;
 import org.apache.flume.conf.Configurables;
 import org.apache.flume.event.EventBuilder;
 import org.apache.kudu.util.DecimalUtil;
-import org.junit.Before;
-import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.kudu.ColumnSchema;
@@ -70,13 +68,17 @@ import org.apache.kudu.client.CreateTableOptions;
 import org.apache.kudu.client.KuduTable;
 
 public class AvroKuduOperationsProducerTest extends BaseKuduTest {
-  private static final String schemaPath = "src/test/avro/testAvroKuduOperationsProducer.avsc";
+  private static String schemaUriString;
   private static String schemaLiteral;
 
   static {
     try {
-      schemaLiteral = Files.toString(new File(schemaPath), UTF_8);
-    } catch (IOException e) {
+      String schemaPath = "/testAvroKuduOperationsProducer.avsc";
+      URL schemaUrl = AvroKuduOperationsProducerTest.class.getResource(schemaPath);
+      File schemaFile = Paths.get(schemaUrl.toURI()).toFile();
+      schemaUriString = schemaFile.getAbsoluteFile().toURI().toString();
+      schemaLiteral = Files.asCharSource(schemaFile, UTF_8).read();
+    } catch (Exception e) {
       throw new RuntimeException(e);
     }
   }
@@ -115,9 +117,8 @@ public class AvroKuduOperationsProducerTest extends BaseKuduTest {
     KuduTable table = createNewTable(
         String.format("test%sevents%s", eventCount, schemaLocation));
     String tableName = table.getName();
-    String schemaURI = new File(schemaPath).getAbsoluteFile().toURI().toString();
     Context ctx = schemaLocation != SchemaLocation.GLOBAL ? new Context()
-        : new Context(ImmutableMap.of(PRODUCER_PREFIX + SCHEMA_PROP, schemaURI));
+        : new Context(ImmutableMap.of(PRODUCER_PREFIX + SCHEMA_PROP, schemaUriString));
     KuduSink sink = createSink(tableName, ctx);
 
     Channel channel = new MemoryChannel();
@@ -190,8 +191,7 @@ public class AvroKuduOperationsProducerTest extends BaseKuduTest {
       encoder.flush();
       Event e = EventBuilder.withBody(out.toByteArray());
       if (schemaLocation == SchemaLocation.URL) {
-        String schemaURI = new File(schemaPath).getAbsoluteFile().toURI().toString();
-        e.setHeaders(ImmutableMap.of(SCHEMA_URL_HEADER, schemaURI));
+        e.setHeaders(ImmutableMap.of(SCHEMA_URL_HEADER, schemaUriString));
       } else if (schemaLocation == SchemaLocation.LITERAL) {
         e.setHeaders(ImmutableMap.of(SCHEMA_LITERAL_HEADER, schemaLiteral));
       }

http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/src/test/resources/testAvroKuduOperationsProducer.avsc
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/test/resources/testAvroKuduOperationsProducer.avsc b/java/kudu-flume-sink/src/test/resources/testAvroKuduOperationsProducer.avsc
new file mode 100644
index 0000000..6bcf6d2
--- /dev/null
+++ b/java/kudu-flume-sink/src/test/resources/testAvroKuduOperationsProducer.avsc
@@ -0,0 +1,14 @@
+{"namespace": "org.apache.kudu.flume.sink",
+  "type": "record",
+  "name": "AvroKuduOperationsProducerTestRecord",
+  "fields": [
+    {"name": "key", "type": "int"},
+    {"name": "longField",  "type": "long"},
+    {"name": "doubleField",  "type": "double"},
+    {"name": "nullableField",  "type": ["string", "null"]},
+    {"name": "stringField", "type": "string"},
+    {"name": "decimalField", "type": {
+        "type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 1}
+    }
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
b/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
index e470ddc..ceca8d1 100644
--- a/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
+++ b/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
@@ -17,7 +17,7 @@
 
 package org.apache.kudu.spark.tools
 
-import java.io.File
+import java.nio.file.Paths
 
 import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder
 import org.apache.kudu.{Schema, Type}
@@ -35,7 +35,7 @@ import scala.collection.JavaConverters._
 class TestImportExportFiles  extends FunSuite with TestContext with  Matchers {
 
   private val TABLE_NAME: String = "TestImportExportFiles"
-  private val TABLE_DATA_PATH: String = "src/test/resources/TestImportExportFiles.csv"
+  private val TABLE_DATA_PATH: String = "/TestImportExportFiles.csv"
 
   test("Spark Import Export") {
     val schema: Schema = {
@@ -51,12 +51,14 @@ class TestImportExportFiles  extends FunSuite with TestContext with  Matchers
{
       .setNumReplicas(1)
     kuduClient.createTable(TABLE_NAME, schema, tableOptions)
 
-    val dataPath = new File(TABLE_DATA_PATH).getAbsolutePath
+    // Get the absolute path of the resource file.
+    val schemaResource = classOf[TestImportExportFiles].getResource(TABLE_DATA_PATH)
+    val dataPath = Paths.get(schemaResource.toURI).toAbsolutePath
 
     ImportExportFiles.testMain(Array("--operation=import",
       "--format=csv",
       s"--master-addrs=${miniCluster.getMasterAddresses}",
-      s"--path=$TABLE_DATA_PATH",
+      s"--path=$dataPath",
       s"--table-name=$TABLE_NAME",
       "--delimiter=,",
       "--header=true",


Mime
View raw message