tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject tajo git commit: TAJO-2001: DirectRawFileScanner.getProgress occasionally fails.
Date Thu, 03 Dec 2015 05:40:31 GMT
Repository: tajo
Updated Branches:
  refs/heads/master 3852ae3d0 -> 8d513a5e8


TAJO-2001: DirectRawFileScanner.getProgress occasionally fails.

Closes #894


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/8d513a5e
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/8d513a5e
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/8d513a5e

Branch: refs/heads/master
Commit: 8d513a5e8c8c8d5b69ba9630329736bb2bb7b86e
Parents: 3852ae3
Author: Jinho Kim <jhkim@apache.org>
Authored: Thu Dec 3 14:39:46 2015 +0900
Committer: Jinho Kim <jhkim@apache.org>
Committed: Thu Dec 3 14:39:46 2015 +0900

----------------------------------------------------------------------
 CHANGES                                         |  2 +
 .../java/org/apache/tajo/storage/RawFile.java   |  5 ++-
 .../storage/rawfile/DirectRawFileScanner.java   | 46 ++++++++------------
 3 files changed, 24 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/8d513a5e/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 688c2f5..9903944 100644
--- a/CHANGES
+++ b/CHANGES
@@ -46,6 +46,8 @@ Release 0.12.0 - unreleased
 
   BUG FIXES
 
+    TAJO-2001: DirectRawFileScanner.getProgress occasionally fails. (jinho)
+
     TAJO-1753: GlobalEngine causes NPE occurs occasionally. (jinho)
 
     TAJO-1862: TSQL gives user wrong URL of documentation. 

http://git-wip-us.apache.org/repos/asf/tajo/blob/8d513a5e/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java
b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java
index 26bd135..f31b85c 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java
@@ -453,10 +453,11 @@ public class RawFile {
         return 1.0f;
       }
 
-      if (filePosition - startOffset == 0) {
+      long readBytes = filePosition - startOffset;
+      if (readBytes == 0) {
         return 0.0f;
       } else {
-        return Math.min(1.0f, ((float) filePosition / endOffset));
+        return Math.min(1.0f, ((float) readBytes / fragment.getLength()));
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/tajo/blob/8d513a5e/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java
b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java
index 550de63..1e2380e 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.io.IOUtils;
 import org.apache.tajo.catalog.Schema;
 import org.apache.tajo.catalog.SchemaUtil;
 import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.statistics.TableStats;
 import org.apache.tajo.conf.TajoConf;
 import org.apache.tajo.plan.expr.EvalNode;
 import org.apache.tajo.storage.*;
@@ -50,7 +51,6 @@ public class DirectRawFileScanner extends FileScanner implements SeekableScanner
   private SeekableInputChannel channel;
 
   private boolean eos = false;
-  private long fileSize;
   private long recordCount;
   private long filePosition;
   private long endOffset;
@@ -95,10 +95,8 @@ public class DirectRawFileScanner extends FileScanner implements SeekableScanner
       }
 
       channel = new LocalFileInputChannel(new FileInputStream(file));
-      fileSize = channel.size();
     } else {
       channel = new FSDataInputChannel(fs.open(fragment.getPath()));
-      fileSize = channel.size();
     }
 
     // initial set position
@@ -106,10 +104,6 @@ public class DirectRawFileScanner extends FileScanner implements SeekableScanner
       channel.seek(fragment.getStartKey());
     }
 
-    if (tableStats != null) {
-      tableStats.setNumBytes(fileSize);
-    }
-
     filePosition = fragment.getStartKey();
     endOffset = fragment.getStartKey() + fragment.getLength();
     if (LOG.isDebugEnabled()) {
@@ -178,7 +172,7 @@ public class DirectRawFileScanner extends FileScanner implements SeekableScanner
   @Override
   public void close() throws IOException {
     if (tableStats != null) {
-      tableStats.setReadBytes(fileSize);
+      tableStats.setReadBytes(filePosition - fragment.getStartKey());
       tableStats.setNumRows(recordCount);
     }
     if(tupleBuffer != null) {
@@ -211,30 +205,28 @@ public class DirectRawFileScanner extends FileScanner implements SeekableScanner
   }
 
   @Override
+  public TableStats getInputStats() {
+    if(tableStats != null){
+      tableStats.setNumRows(recordCount);
+      tableStats.setReadBytes(filePosition - fragment.getStartKey()); // actual read bytes
(scan + rescan * n)
+      tableStats.setNumBytes(fragment.getLength());
+    }
+    return tableStats;
+  }
+
+  @Override
   public float getProgress() {
     if(!inited) return 0.0f;
 
-    try {
-      tableStats.setNumRows(recordCount);
-      long filePos = 0;
-      if (channel != null) {
-        filePos = channel.position();
-        tableStats.setReadBytes(filePos);
-      }
-
-      if(eos || channel == null) {
-        tableStats.setReadBytes(fileSize);
-        return 1.0f;
-      }
+    if(eos) {
+      return 1.0f;
+    }
 
-      if (filePos == 0) {
-        return 0.0f;
-      } else {
-        return Math.min(1.0f, ((float)filePos / (float)fileSize));
-      }
-    } catch (IOException e) {
-      LOG.error(e.getMessage(), e);
+    long readBytes = filePosition - fragment.getStartKey();
+    if (readBytes == 0) {
       return 0.0f;
+    } else {
+      return Math.min(1.0f, ((float) readBytes / fragment.getLength()));
     }
   }
 }


Mime
View raw message