incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Fixing issue with truncated lastmod companion files and using SegmentInfoPerCommit for files in a commited segment instead of SegmentInfo.
Date Tue, 28 Apr 2015 00:54:51 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master 8e36828b0 -> 0e4e324ac


Fixing issue with truncated lastmod companion files and using SegmentInfoPerCommit for files
in a commited segment instead of SegmentInfo.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/0e4e324a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/0e4e324a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/0e4e324a

Branch: refs/heads/master
Commit: 0e4e324acf064d6e9cc11e2fb741ad715cb54673
Parents: 8e36828
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Apr 27 20:54:48 2015 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Apr 27 20:54:48 2015 -0400

----------------------------------------------------------------------
 .../apache/blur/mapreduce/lib/GenericRecordReader.java   | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e4e324a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/GenericRecordReader.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/GenericRecordReader.java
b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/GenericRecordReader.java
index 259bf72..4b9b526 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/GenericRecordReader.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/GenericRecordReader.java
@@ -17,7 +17,7 @@
 package org.apache.blur.mapreduce.lib;
 
 import java.io.IOException;
-import java.util.Set;
+import java.util.Collection;
 
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
@@ -76,7 +76,7 @@ public class GenericRecordReader {
     SegmentInfo segmentInfo = commit.info;
     if (localCachePath != null) {
       _readingDirectory = copyFilesLocally(configuration, _directory, _table.toString(),
blurInputSplit.getDir(),
-          localCachePath, segmentInfo.files());
+          localCachePath, commit.files());
     } else {
       _readingDirectory = _directory;
     }
@@ -97,7 +97,7 @@ public class GenericRecordReader {
   }
 
   private static Directory copyFilesLocally(Configuration configuration, Directory dir, String
table, Path shardDir,
-      Path localCachePath, Set<String> files) throws IOException {
+      Path localCachePath, Collection<String> files) throws IOException {
     LOG.info("Copying files need to local cache for faster reads [{0}].", shardDir);
     Path localShardPath = new Path(new Path(localCachePath, table), shardDir.getName());
     HdfsDirectory localDir = new HdfsDirectory(configuration, localShardPath);
@@ -145,9 +145,10 @@ public class GenericRecordReader {
       LOG.info("Cache file exists [{0}]", name);
       if (localDir.fileLength(name) == fileLength) {
         LOG.info("Cache file length matches [{0}]", name);
-        if (localDir.fileExists(name + LASTMOD)) {
+        String lastModFile = name + LASTMOD;
+        if (localDir.fileExists(lastModFile) && localDir.fileLength(lastModFile)
!= 8) {
           LOG.info("Cache file last mod file exists [{0}]", name);
-          IndexInput input = localDir.openInput(name + LASTMOD, IOContext.DEFAULT);
+          IndexInput input = localDir.openInput(lastModFile, IOContext.DEFAULT);
           long lastMod = input.readLong();
           if (lastMod == fileModified) {
             LOG.info("Cache file last mod matches [{0}]", name);


Mime
View raw message