tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kshu...@apache.org
Subject tez git commit: TEZ-3723. TezIndexRecord#hasData() returns true for empty index record in the Unordered case (kshukla)
Date Tue, 16 May 2017 21:31:33 GMT
Repository: tez
Updated Branches:
  refs/heads/master 80f95b1d6 -> e3ee7a63b


TEZ-3723. TezIndexRecord#hasData() returns true for empty index record in the Unordered case
(kshukla)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/e3ee7a63
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/e3ee7a63
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/e3ee7a63

Branch: refs/heads/master
Commit: e3ee7a63b0289868db535008ed3c4a5d2fc6c4e3
Parents: 80f95b1
Author: Kuhu Shukla <kshukla@yahoo-inc.com>
Authored: Tue May 16 16:25:33 2017 -0500
Committer: Kuhu Shukla <kshukla@yahoo-inc.com>
Committed: Tue May 16 16:25:33 2017 -0500

----------------------------------------------------------------------
 .../tez/runtime/library/common/sort/impl/TezIndexRecord.java    | 2 +-
 .../tez/runtime/library/common/shuffle/TestShuffleUtils.java    | 5 ++---
 .../common/writers/TestUnorderedPartitionedKVWriter.java        | 3 ++-
 3 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/e3ee7a63/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezIndexRecord.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezIndexRecord.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezIndexRecord.java
index 2f04a4d..872cd10 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezIndexRecord.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezIndexRecord.java
@@ -51,6 +51,6 @@ public class TezIndexRecord {
   public boolean hasData() {
     //TEZ-941 - Avoid writing out empty partitions
     //EOF_MARKER + Header bytes
-    return !(rawLength == (IFile.HEADER.length + 2));
+    return !(rawLength <= (IFile.HEADER.length + 2));
   }
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/e3ee7a63/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java
index b1ce716..da09324 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java
@@ -43,8 +43,8 @@ import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.BitSet;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.List;
-import java.util.Random;
 
 import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.anyInt;
@@ -137,9 +137,8 @@ public class TestShuffleUtils {
     TezSpillRecord spillRecord = new TezSpillRecord(numPartitions);
     long startOffset = 0;
     long partLen = 200; //compressed
-    Random rnd = new Random();
     for(int i=0;i<numPartitions;i++) {
-      long rawLen = rnd.nextLong();
+      long rawLen = ThreadLocalRandom.current().nextLong(100, 200);
       if (i % 2  == 0 || allEmptyPartitions) {
         rawLen = 6; //indicates empty partition
       }

http://git-wip-us.apache.org/repos/asf/tez/blob/e3ee7a63/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java
index 5cda126..6ea1562 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java
@@ -933,10 +933,11 @@ public class TestUnorderedPartitionedKVWriter {
     IntWritable keyDeser = new IntWritable();
     LongWritable valDeser = new LongWritable();
     for (int i = 0; i < numOutputs; i++) {
+      TezIndexRecord indexRecord = spillRecord.getIndex(i);
       if (skippedPartitions != null && skippedPartitions.contains(i)) {
+        assertFalse("The Index Record for partition " + i + " should not have any data",
indexRecord.hasData());
         continue;
       }
-      TezIndexRecord indexRecord = spillRecord.getIndex(i);
       FSDataInputStream inStream = FileSystem.getLocal(conf).open(outputFilePath);
       inStream.seek(indexRecord.getStartOffset());
       IFile.Reader reader = new IFile.Reader(inStream, indexRecord.getPartLength(), codec,
null,


Mime
View raw message