Return-Path: X-Original-To: apmail-tajo-commits-archive@minotaur.apache.org Delivered-To: apmail-tajo-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E604511756 for ; Tue, 15 Jul 2014 07:16:41 +0000 (UTC) Received: (qmail 2041 invoked by uid 500); 15 Jul 2014 07:16:41 -0000 Delivered-To: apmail-tajo-commits-archive@tajo.apache.org Received: (qmail 1951 invoked by uid 500); 15 Jul 2014 07:16:41 -0000 Mailing-List: contact commits-help@tajo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tajo.apache.org Delivered-To: mailing list commits@tajo.apache.org Received: (qmail 1659 invoked by uid 99); 15 Jul 2014 07:16:41 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Jul 2014 07:16:41 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 89A8894BDE7; Tue, 15 Jul 2014 07:16:41 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jihoonson@apache.org To: commits@tajo.apache.org Date: Tue, 15 Jul 2014 07:16:55 -0000 Message-Id: In-Reply-To: <15a944093b504f69802422dbee053d75@git.apache.org> References: <15a944093b504f69802422dbee053d75@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [16/19] git commit: TAJO-936: TestStorages::testSplitable is failed occasionally. (jinho) TAJO-936: TestStorages::testSplitable is failed occasionally. (jinho) Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/aee7874f Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/aee7874f Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/aee7874f Branch: refs/heads/index_support Commit: aee7874faa45e3ba2eda3157683cf20111cacea5 Parents: 48dbfd9 Author: jinossy Authored: Tue Jul 15 14:25:51 2014 +0900 Committer: jinossy Committed: Tue Jul 15 14:25:51 2014 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../org/apache/tajo/storage/rcfile/RCFile.java | 1 + .../org/apache/tajo/storage/TestStorages.java | 58 +++++++++++++++++++- 3 files changed, 60 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/aee7874f/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 3803ded..473db19 100644 --- a/CHANGES +++ b/CHANGES @@ -84,6 +84,8 @@ Release 0.9.0 - unreleased BUG FIXES + TAJO-936: TestStorages::testSplitable is failed occasionally. (jinho) + TAJO-673: Assign proper number of tasks when inserting into partitioned table. (jaehwa) TAJO-916: SubQuery::computeStatFromTasks occasionally fail. http://git-wip-us.apache.org/repos/asf/tajo/blob/aee7874f/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java index 1da41e4..4cc37b3 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java @@ -1380,6 +1380,7 @@ public class RCFile { in.readFully(sync); // read sync bytes headerEnd = in.getPos(); + lastSeenSyncPos = headerEnd; //initial sync position readBytes += headerEnd; } http://git-wip-us.apache.org/repos/asf/tajo/blob/aee7874f/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java index 944aa5c..1cf1ecf 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java @@ -23,7 +23,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.QueryId; import org.apache.tajo.TajoIdProtos; -import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes.Type; @@ -175,6 +177,60 @@ public class TestStorages { } @Test + public void testRCFileSplitable() throws IOException { + if (storeType == StoreType.RCFILE) { + Schema schema = new Schema(); + schema.addColumn("id", Type.INT4); + schema.addColumn("age", Type.INT8); + + TableMeta meta = CatalogUtil.newTableMeta(storeType); + Path tablePath = new Path(testDir, "Splitable.data"); + Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath); + appender.enableStats(); + appender.init(); + int tupleNum = 10000; + VTuple vTuple; + + for (int i = 0; i < tupleNum; i++) { + vTuple = new VTuple(2); + vTuple.put(0, DatumFactory.createInt4(i + 1)); + vTuple.put(1, DatumFactory.createInt8(25l)); + appender.addTuple(vTuple); + } + appender.close(); + TableStats stat = appender.getStats(); + assertEquals(tupleNum, stat.getNumRows().longValue()); + + FileStatus status = fs.getFileStatus(tablePath); + long fileLen = status.getLen(); + long randomNum = 122; // header size + + FileFragment[] tablets = new FileFragment[2]; + tablets[0] = new FileFragment("Splitable", tablePath, 0, randomNum); + tablets[1] = new FileFragment("Splitable", tablePath, randomNum, (fileLen - randomNum)); + + Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema); + assertTrue(scanner.isSplittable()); + scanner.init(); + int tupleCnt = 0; + while (scanner.next() != null) { + tupleCnt++; + } + scanner.close(); + + scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[1], schema); + assertTrue(scanner.isSplittable()); + scanner.init(); + while (scanner.next() != null) { + tupleCnt++; + } + scanner.close(); + + assertEquals(tupleNum, tupleCnt); + } + } + + @Test public void testProjection() throws IOException { Schema schema = new Schema(); schema.addColumn("id", Type.INT4);