tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject tajo git commit: TAJO-2016: Investigate storage tests hangs.
Date Thu, 10 Dec 2015 06:48:23 GMT
Repository: tajo
Updated Branches:
  refs/heads/master 3d59eebd5 -> 394879e0f


TAJO-2016: Investigate storage tests hangs.

Closes #906


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/394879e0
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/394879e0
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/394879e0

Branch: refs/heads/master
Commit: 394879e0f01273b2c8a66aef01d6cc313a5e5f3d
Parents: 3d59eeb
Author: Jinho Kim <jhkim@apache.org>
Authored: Thu Dec 10 15:46:33 2015 +0900
Committer: Jinho Kim <jhkim@apache.org>
Committed: Thu Dec 10 15:46:33 2015 +0900

----------------------------------------------------------------------
 CHANGES                                         |  2 ++
 .../tajo/storage/TestByteBufLineReader.java     |  7 +++---
 .../tajo/storage/TestCompressionStorages.java   | 15 ++++++------
 .../apache/tajo/storage/TestFileTablespace.java | 25 ++++++++++----------
 .../tajo/storage/raw/TestDirectRawFile.java     |  2 +-
 5 files changed, 27 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/394879e0/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index c859b18..25a4791 100644
--- a/CHANGES
+++ b/CHANGES
@@ -57,6 +57,8 @@ Release 0.12.0 - unreleased
 
   BUG FIXES
 
+    TAJO-2016: Fix storage tests hangs. (jinho)
+
     TAJO-2013: FilterPushDownRule fails due to the unsupported index. (jinho)
 
     TAJO-2014: TestRpcClientManager fails occasionally. (jinho)

http://git-wip-us.apache.org/repos/asf/tajo/blob/394879e0/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
index b6f65df..8472095 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
@@ -85,7 +85,7 @@ public class TestByteBufLineReader {
     fs.close();
   }
 
-  @Test
+  @Test(timeout = 60000)
   public void testReaderWithDFS() throws Exception {
     final Configuration conf = new HdfsConfiguration();
     String testDataPath = TEST_PATH + "/" + UUID.randomUUID().toString();
@@ -94,8 +94,7 @@ public class TestByteBufLineReader {
     conf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, true);
 
     final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
-        .numDataNodes(2).build();
-    cluster.waitClusterUp();
+        .numDataNodes(2).waitSafeMode(true).build();
 
     TajoConf tajoConf = new TajoConf(conf);
     tajoConf.setVar(TajoConf.ConfVars.ROOT_DIR, cluster.getFileSystem().getUri() + "/tajo");
@@ -122,7 +121,7 @@ public class TestByteBufLineReader {
       lineReader.close();
       fs.close();
     } finally {
-      cluster.shutdown(true);
+      cluster.shutdown();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/394879e0/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
index 7836730..b63b497 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.compress.*;
 import org.apache.hadoop.io.compress.zlib.ZlibFactory;
 import org.apache.hadoop.util.NativeCodeLoader;
+import org.apache.tajo.BuiltinStorages;
 import org.apache.tajo.catalog.CatalogUtil;
 import org.apache.tajo.catalog.Schema;
 import org.apache.tajo.catalog.TableMeta;
@@ -68,9 +69,9 @@ public class TestCompressionStorages {
   @Parameterized.Parameters
   public static Collection<Object[]> generateParameters() {
     return Arrays.asList(new Object[][]{
-        {"TEXT"},
-        {"RCFILE"},
-        {"SEQUENCEFILE"}
+        {BuiltinStorages.TEXT},
+        {BuiltinStorages.RCFILE},
+        {BuiltinStorages.SEQUENCE_FILE}
     });
   }
 
@@ -81,11 +82,11 @@ public class TestCompressionStorages {
 
   @Test
   public void testGzipCodecCompressionData() throws IOException {
-    if (dataFormat.equalsIgnoreCase("RCFILE")) {
+    if (dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) {
       if( ZlibFactory.isNativeZlibLoaded(conf)) {
         storageCompressionTest(dataFormat, GzipCodec.class);
       }
-    } else if (dataFormat.equalsIgnoreCase("SEQUENCEFILE")) {
+    } else if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) {
       if( ZlibFactory.isNativeZlibLoaded(conf)) {
         storageCompressionTest(dataFormat, GzipCodec.class);
       }
@@ -131,7 +132,7 @@ public class TestCompressionStorages {
       extension = ((DelimitedTextFile.DelimitedTextFileAppender) appender).getExtension();
     }
 
-    int tupleNum = 100000;
+    int tupleNum = 1000;
     VTuple vTuple;
 
     for (int i = 0; i < tupleNum; i++) {
@@ -154,7 +155,7 @@ public class TestCompressionStorages {
     Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, tablets[0],
schema);
     scanner.init();
 
-    if (dataFormat.equalsIgnoreCase("SEQUENCEFILE")) {
+    if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) {
       assertTrue(scanner instanceof SequenceFileScanner);
       Writable key = ((SequenceFileScanner) scanner).getKey();
       assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());

http://git-wip-us.apache.org/repos/asf/tajo/blob/394879e0/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
index d599a25..c99e0dc 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.*;
+import org.apache.tajo.BuiltinStorages;
 import org.apache.tajo.catalog.CatalogUtil;
 import org.apache.tajo.catalog.Schema;
 import org.apache.tajo.catalog.TableMeta;
@@ -41,7 +42,6 @@ import org.junit.Test;
 import java.io.IOException;
 import java.net.URI;
 import java.util.List;
-import java.util.Optional;
 import java.util.UUID;
 
 import static org.junit.Assert.*;
@@ -70,7 +70,7 @@ public class TestFileTablespace {
 		schema.addColumn("age",Type.INT4);
 		schema.addColumn("name",Type.TEXT);
 
-		TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+		TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT);
 		
 		VTuple[] tuples = new VTuple[4];
 		for(int i=0; i < tuples.length; i++) {
@@ -82,7 +82,7 @@ public class TestFileTablespace {
 
     Path path = StorageUtil.concatPath(testDir, "testGetScannerAndAppender", "table.csv");
     localFs.mkdirs(path.getParent());
-    FileTablespace fileStorageManager = (FileTablespace) TablespaceManager.getLocalFs();
+    FileTablespace fileStorageManager = TablespaceManager.getLocalFs();
     assertEquals(localFs.getUri(), fileStorageManager.getFileSystem().getUri());
 
 		Appender appender = fileStorageManager.getAppender(meta, schema, path);
@@ -99,6 +99,7 @@ public class TestFileTablespace {
 			i++;
 		}
 		assertEquals(4,i);
+    localFs.delete(path, true);
 	}
 
   @Test(timeout = 60000)
@@ -137,7 +138,7 @@ public class TestFileTablespace {
       schema.addColumn("id", Type.INT4);
       schema.addColumn("age",Type.INT4);
       schema.addColumn("name",Type.TEXT);
-      TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+      TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT);
 
       List<Fragment> splits = Lists.newArrayList();
       // Get FileFragments in partition batch
@@ -154,7 +155,7 @@ public class TestFileTablespace {
       assertEquals(-1, ((FileFragment)splits.get(0)).getDiskIds()[0]);
       fs.close();
     } finally {
-      cluster.shutdown(true);
+      cluster.shutdown();
     }
   }
 
@@ -203,7 +204,7 @@ public class TestFileTablespace {
       assertEquals(0, splits.size());
       fs.close();
     } finally {
-      cluster.shutdown(true);
+      cluster.shutdown();
     }
   }
 
@@ -254,7 +255,7 @@ public class TestFileTablespace {
       assertNotEquals(-1, ((FileFragment)splits.get(0)).getDiskIds()[0]);
       fs.close();
     } finally {
-      cluster.shutdown(true);
+      cluster.shutdown();
     }
   }
 
@@ -271,7 +272,6 @@ public class TestFileTablespace {
         new MiniDFSCluster.Builder(hdfsConf).numDataNodes(1).build();
     URI uri = URI.create(cluster.getFileSystem().getUri() + "/tajo");
 
-    Optional<Tablespace> existingTs = Optional.empty();
     try {
       /* Local FileSystem */
       FileTablespace space = TablespaceManager.getLocalFs();
@@ -279,16 +279,17 @@ public class TestFileTablespace {
 
       FileTablespace distTablespace = new FileTablespace("testGetFileTablespace", uri, null);
       distTablespace.init(conf);
-      existingTs = TablespaceManager.addTableSpaceForTest(distTablespace);
+
+      TablespaceManager.addTableSpaceForTest(distTablespace);
 
       /* Distributed FileSystem */
-      space = (FileTablespace) TablespaceManager.get(uri);
+      space = TablespaceManager.get(uri);
       assertEquals(cluster.getFileSystem().getUri(), space.getFileSystem().getUri());
 
-      space = (FileTablespace) TablespaceManager.getByName("testGetFileTablespace");
+      space = TablespaceManager.getByName("testGetFileTablespace");
       assertEquals(cluster.getFileSystem().getUri(), space.getFileSystem().getUri());
     } finally {
-      cluster.shutdown(true);
+      cluster.shutdown();
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/tajo/blob/394879e0/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
index 179c9eb..78e3390 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
@@ -117,7 +117,7 @@ public class TestDirectRawFile {
 
   @AfterClass
   public static void tearDownClass() throws InterruptedException {
-    cluster.shutdown(true);
+    cluster.shutdown();
   }
 
   public Path getTestDir(FileSystem fs, String dir) throws IOException {


Mime
View raw message