kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From granthe...@apache.org
Subject kudu git commit: [Java] Remove hard coded usage of /target directory
Date Mon, 23 Apr 2018 18:15:20 GMT
Repository: kudu
Updated Branches:
  refs/heads/master 650e39e23 -> 08b01ecf3


[Java] Remove hard coded usage of /target directory

This causes tests to fail in Gradle.

Change-Id: Ic6a27e2b049a5a25cee070edd8c7d57d55f177e8
Reviewed-on: http://gerrit.cloudera.org:8080/10154
Reviewed-by: Will Berkeley <wdberkeley@gmail.com>
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/08b01ecf
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/08b01ecf
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/08b01ecf

Branch: refs/heads/master
Commit: 08b01ecf30e257f822bdb318424c2288cf693b96
Parents: 650e39e
Author: Grant Henke <granthenke@apache.org>
Authored: Mon Apr 23 11:37:50 2018 -0500
Committer: Grant Henke <granthenke@apache.org>
Committed: Mon Apr 23 18:14:55 2018 +0000

----------------------------------------------------------------------
 .../org/apache/kudu/mapreduce/JarFinder.java    | 20 +++++++-
 .../kudu/mapreduce/HadoopTestingUtility.java    | 13 ++++-
 .../apache/kudu/mapreduce/TestJarFinder.java    | 51 ++++++++------------
 .../test/resources/TestImportExportFiles.csv    |  5 ++
 .../spark/tools/TestImportExportFiles.scala     | 21 ++------
 5 files changed, 59 insertions(+), 51 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/08b01ecf/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java
index e1d6a0c..5431fc8 100644
--- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java
@@ -25,6 +25,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
 import java.net.URLDecoder;
+import java.nio.file.Files;
 import java.text.MessageFormat;
 import java.util.Enumeration;
 import java.util.jar.JarFile;
@@ -34,6 +35,7 @@ import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
 import com.google.common.base.Preconditions;
+import org.apache.yetus.audience.InterfaceAudience;
 
 /**
  * Finds the Jar for a class. If the class is in a directory in the
@@ -45,6 +47,11 @@ import com.google.common.base.Preconditions;
  */
 public class JarFinder {
 
+  @InterfaceAudience.Private
+  public static String FILE_DIR_PROPERTY = "jar.file.dir";
+
+  private static File fileDir = null;
+
   private static void copyToZipStream(File file, ZipEntry entry,
                                       ZipOutputStream zos) throws IOException {
     InputStream is = new FileInputStream(file);
@@ -155,7 +162,7 @@ public class JarFinder {
             String klassName = klass.getName();
             klassName = klassName.replace(".", "/") + ".class";
             path = path.substring(0, path.length() - klassName.length());
-            File testDir = new File(System.getProperty("test.build.dir", "target/test-dir"));
+            File testDir = getFileDir();
             testDir = testDir.getAbsoluteFile();
             if (!testDir.exists()) {
               testDir.mkdirs();
@@ -174,4 +181,15 @@ public class JarFinder {
     }
     return null;
   }
+
+  private static File getFileDir() throws IOException {
+    if (fileDir == null) {
+      String testDirPath = System.getProperty(FILE_DIR_PROPERTY);
+      if (testDirPath == null) {
+        testDirPath = Files.createTempDirectory("file-dir").toString();
+      }
+      fileDir = new File(testDirPath);
+    }
+    return fileDir;
+  }
 }

http://git-wip-us.apache.org/repos/asf/kudu/blob/08b01ecf/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java
b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java
index c8ced88..e9cdf06 100644
--- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java
@@ -20,6 +20,7 @@ package org.apache.kudu.mapreduce;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.logging.Log;
@@ -48,7 +49,7 @@ public class HadoopTestingUtility {
   /**
    * Default base directory for test output.
    */
-  private static final String DEFAULT_BASE_TEST_DIRECTORY = "target/mr-data";
+  private static final String DEFAULT_BASE_TEST_DIRECTORY_PREFIX = "mr-data";
 
   public Configuration getConfiguration() {
     return this.conf;
@@ -77,7 +78,15 @@ public class HadoopTestingUtility {
   }
 
   private Path getBaseTestDir() {
-    String pathName = System.getProperty(BASE_TEST_DIRECTORY_KEY, DEFAULT_BASE_TEST_DIRECTORY);
+    String pathName = System.getProperty(BASE_TEST_DIRECTORY_KEY);
+    // Use a temporary by default if no test directory is specified.
+    if (pathName == null) {
+      try {
+        pathName = Files.createTempDirectory(DEFAULT_BASE_TEST_DIRECTORY_PREFIX).toString();
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
     return new Path(pathName);
   }
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/08b01ecf/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java
b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java
index 710d5a1..5f640e0 100644
--- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java
@@ -21,17 +21,19 @@ import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.FileWriter;
-import java.io.IOException;
 import java.io.OutputStream;
 import java.io.Writer;
-import java.text.MessageFormat;
+import java.nio.file.Files;
 import java.util.Properties;
 import java.util.jar.JarInputStream;
 import java.util.jar.JarOutputStream;
 import java.util.jar.Manifest;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.logging.LogFactory;
+import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 /**
@@ -39,37 +41,26 @@ import org.junit.Test;
  */
 public class TestJarFinder {
 
+  private static File testDir;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    testDir = Files.createTempDirectory("test-dir").toFile();
+    System.setProperty(JarFinder.FILE_DIR_PROPERTY, testDir.getAbsolutePath());
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    FileUtils.deleteDirectory(testDir);
+  }
+
   @Test
   public void testJar() throws Exception {
-
     // Picking a class that is for sure in a JAR in the classpath
     String jar = JarFinder.getJar(LogFactory.class);
     Assert.assertTrue(new File(jar).exists());
   }
 
-  private static void delete(File file) throws IOException {
-    if (file.getAbsolutePath().length() < 5) {
-      throw new IllegalArgumentException(
-        MessageFormat.format("Path [{0}] is too short, not deleting",
-          file.getAbsolutePath()));
-    }
-    if (file.exists()) {
-      if (file.isDirectory()) {
-        File[] children = file.listFiles();
-        if (children != null) {
-          for (File child : children) {
-            delete(child);
-          }
-        }
-      }
-      if (!file.delete()) {
-        throw new RuntimeException(
-          MessageFormat.format("Could not delete path [{0}]",
-            file.getAbsolutePath()));
-      }
-    }
-  }
-
   @Test
   public void testExpandedClasspath() throws Exception {
     // Picking a class that is for sure in a directory in the classpath
@@ -80,11 +71,8 @@ public class TestJarFinder {
 
   @Test
   public void testExistingManifest() throws Exception {
-    File dir = new File(System.getProperty("test.build.dir", "target/test-dir"),
+    File dir = new File(testDir,
       TestJarFinder.class.getName() + "-testExistingManifest");
-    delete(dir);
-    dir.mkdirs();
-
     File metaInfDir = new File(dir, "META-INF");
     metaInfDir.mkdirs();
     File manifestFile = new File(metaInfDir, "MANIFEST.MF");
@@ -108,9 +96,8 @@ public class TestJarFinder {
 
   @Test
   public void testNoManifest() throws Exception {
-    File dir = new File(System.getProperty("test.build.dir", "target/test-dir"),
+    File dir = new File(testDir,
       TestJarFinder.class.getName() + "-testNoManifest");
-    delete(dir);
     dir.mkdirs();
     File propsFile = new File(dir, "props.properties");
     Writer writer = new FileWriter(propsFile);

http://git-wip-us.apache.org/repos/asf/kudu/blob/08b01ecf/java/kudu-spark-tools/src/test/resources/TestImportExportFiles.csv
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/src/test/resources/TestImportExportFiles.csv b/java/kudu-spark-tools/src/test/resources/TestImportExportFiles.csv
new file mode 100644
index 0000000..22144d2
--- /dev/null
+++ b/java/kudu-spark-tools/src/test/resources/TestImportExportFiles.csv
@@ -0,0 +1,5 @@
+key,column1_i,column2_d,column3_s,column4_b
+1,3,2.3,some string,true
+2,5,4.5,some more,false
+3,7,1.2,wait this is not a double bad row,true
+4,9,10.1,trailing separator isn't bad mkay?,true
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kudu/blob/08b01ecf/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
b/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
index 1d4d314..e470ddc 100644
--- a/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
+++ b/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
@@ -17,7 +17,7 @@
 
 package org.apache.kudu.spark.tools
 
-import java.io.{File, FileOutputStream}
+import java.io.File
 
 import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder
 import org.apache.kudu.{Schema, Type}
@@ -34,7 +34,8 @@ import scala.collection.JavaConverters._
 @RunWith(classOf[JUnitRunner])
 class TestImportExportFiles  extends FunSuite with TestContext with  Matchers {
 
-  private val TABLE_NAME: String = classOf[TestImportExportFiles].getName + "-" + System.currentTimeMillis
+  private val TABLE_NAME: String = "TestImportExportFiles"
+  private val TABLE_DATA_PATH: String = "src/test/resources/TestImportExportFiles.csv"
 
   test("Spark Import Export") {
     val schema: Schema = {
@@ -50,13 +51,12 @@ class TestImportExportFiles  extends FunSuite with TestContext with  Matchers
{
       .setNumReplicas(1)
     kuduClient.createTable(TABLE_NAME, schema, tableOptions)
 
-    val data: File = new File("target/", TABLE_NAME+".csv")
-    writeCsvFile(data)
+    val dataPath = new File(TABLE_DATA_PATH).getAbsolutePath
 
     ImportExportFiles.testMain(Array("--operation=import",
       "--format=csv",
       s"--master-addrs=${miniCluster.getMasterAddresses}",
-      s"--path=${"target/"+TABLE_NAME+".csv"}",
+      s"--path=$TABLE_DATA_PATH",
       s"--table-name=$TABLE_NAME",
       "--delimiter=,",
       "--header=true",
@@ -65,15 +65,4 @@ class TestImportExportFiles  extends FunSuite with TestContext with  Matchers
{
     assert(rdd.collect.length == 4)
     assertEquals(rdd.collect().mkString(","),"[1],[2],[3],[4]")
   }
-
-  def writeCsvFile(data: File)
-  {
-    val fos: FileOutputStream = new FileOutputStream(data)
-    fos.write("key,column1_i,column2_d,column3_s,column4_b\n".getBytes)
-    fos.write("1,3,2.3,some string,true\n".getBytes)
-    fos.write("2,5,4.5,some more,false\n".getBytes)
-    fos.write("3,7,1.2,wait this is not a double bad row,true\n".getBytes)
-    fos.write("4,9,10.1,trailing separator isn't bad mkay?,true\n".getBytes)
-    fos.close()
-  }
 }


Mime
View raw message