incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [01/29] git commit: Adding a log message to know when progress is setup correctly and changed the progress index output to a buffered version so not to miss and progress events.
Date Fri, 27 Jun 2014 13:25:44 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/apache-blur-0.2 f935941c4 -> 70ce3b5d5


Adding a log message to know when progress is setup correctly and changed the progress index
output to a buffered version so not to miss and progress events.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/00007c78
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/00007c78
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/00007c78

Branch: refs/heads/apache-blur-0.2
Commit: 00007c78d1ed5ecf58269867cf28baeea86b0d02
Parents: f935941
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Fri Jun 27 09:22:32 2014 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Fri Jun 27 09:22:32 2014 -0400

----------------------------------------------------------------------
 .../mapreduce/lib/GenericBlurRecordWriter.java  |  5 ++
 .../mapreduce/lib/ProgressableDirectory.java    | 80 ++------------------
 .../mapreduce/lib/GenericBlurRecordWriter.java  |  5 ++
 .../mapreduce/lib/ProgressableDirectory.java    | 80 ++------------------
 4 files changed, 24 insertions(+), 146 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/00007c78/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
index ebe9a7b..0a4544c 100644
--- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
+++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
@@ -142,11 +142,16 @@ public class GenericBlurRecordWriter {
 
       private Progressable _progressable = prg;
       private long _lastWarn = 0;
+      private boolean _progressSetupLogged = false;
 
       @Override
       public void progress() {
         if (_progressable != null) {
           _progressable.progress();
+          if (!_progressSetupLogged) {
+            LOG.info("Progress has been setup correctly.");
+            _progressSetupLogged = true;
+          }
         } else {
           Progressable progressable = BlurOutputFormat.getProgressable();
           if (progressable != null) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/00007c78/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
index 004e1fa..15700bc 100644
--- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
+++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
@@ -18,13 +18,12 @@ package org.apache.blur.mapreduce.lib;
  */
 import java.io.IOException;
 import java.util.Collection;
-import java.util.Map;
 
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.hadoop.util.Progressable;
 import org.apache.lucene.store.BufferedIndexInput;
-import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.BufferedIndexOutput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
@@ -146,8 +145,7 @@ public class ProgressableDirectory extends Directory {
     return _directory.toString();
   }
 
-  @SuppressWarnings("deprecation")
-  static class ProgressableIndexOutput extends IndexOutput {
+  static class ProgressableIndexOutput extends BufferedIndexOutput {
 
     private Progressable _progressable;
     private IndexOutput _indexOutput;
@@ -158,87 +156,23 @@ public class ProgressableDirectory extends Directory {
     }
 
     @Override
-    public void close() throws IOException {
-      _indexOutput.close();
-      _progressable.progress();
-    }
-
-    @Override
-    public void copyBytes(DataInput input, long numBytes) throws IOException {
-      _indexOutput.copyBytes(input, numBytes);
+    protected void flushBuffer(byte[] b, int offset, int len) throws IOException {
+      _indexOutput.writeBytes(b, offset, len);
       _progressable.progress();
     }
 
     @Override
-    public void flush() throws IOException {
-      _indexOutput.flush();
-      _progressable.progress();
-    }
-
-    @Override
-    public long getFilePointer() {
-      return _indexOutput.getFilePointer();
-    }
-
-    @Override
     public long length() throws IOException {
       return _indexOutput.length();
     }
 
     @Override
-    public void seek(long pos) throws IOException {
-      _indexOutput.seek(pos);
-      _progressable.progress();
-    }
-
-    @Override
-    public void setLength(long length) throws IOException {
-      _indexOutput.setLength(length);
-      _progressable.progress();
-    }
-
-    @Override
-    public String toString() {
-      return _indexOutput.toString();
-    }
-
-    @Override
-    public void writeByte(byte b) throws IOException {
-      _indexOutput.writeByte(b);
-    }
-
-    @Override
-    public void writeBytes(byte[] b, int offset, int length) throws IOException {
-      _indexOutput.writeBytes(b, offset, length);
-      _progressable.progress();
-    }
-
-    @Override
-    public void writeBytes(byte[] b, int length) throws IOException {
-      _indexOutput.writeBytes(b, length);
+    public void close() throws IOException {
+      super.close();
+      _indexOutput.close();
       _progressable.progress();
     }
 
-    @Override
-    public void writeInt(int i) throws IOException {
-      _indexOutput.writeInt(i);
-    }
-
-    @Override
-    public void writeLong(long i) throws IOException {
-      _indexOutput.writeLong(i);
-    }
-
-    @Override
-    public void writeString(String s) throws IOException {
-      _indexOutput.writeString(s);
-    }
-
-    @Override
-    public void writeStringStringMap(Map<String, String> map) throws IOException {
-      _indexOutput.writeStringStringMap(map);
-    }
-
   }
 
   static class ProgressableIndexInput extends BufferedIndexInput {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/00007c78/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
index ebe9a7b..0a4544c 100644
--- a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
+++ b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
@@ -142,11 +142,16 @@ public class GenericBlurRecordWriter {
 
       private Progressable _progressable = prg;
       private long _lastWarn = 0;
+      private boolean _progressSetupLogged = false;
 
       @Override
       public void progress() {
         if (_progressable != null) {
           _progressable.progress();
+          if (!_progressSetupLogged) {
+            LOG.info("Progress has been setup correctly.");
+            _progressSetupLogged = true;
+          }
         } else {
           Progressable progressable = BlurOutputFormat.getProgressable();
           if (progressable != null) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/00007c78/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
index 004e1fa..15700bc 100644
--- a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
+++ b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
@@ -18,13 +18,12 @@ package org.apache.blur.mapreduce.lib;
  */
 import java.io.IOException;
 import java.util.Collection;
-import java.util.Map;
 
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.hadoop.util.Progressable;
 import org.apache.lucene.store.BufferedIndexInput;
-import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.BufferedIndexOutput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
@@ -146,8 +145,7 @@ public class ProgressableDirectory extends Directory {
     return _directory.toString();
   }
 
-  @SuppressWarnings("deprecation")
-  static class ProgressableIndexOutput extends IndexOutput {
+  static class ProgressableIndexOutput extends BufferedIndexOutput {
 
     private Progressable _progressable;
     private IndexOutput _indexOutput;
@@ -158,87 +156,23 @@ public class ProgressableDirectory extends Directory {
     }
 
     @Override
-    public void close() throws IOException {
-      _indexOutput.close();
-      _progressable.progress();
-    }
-
-    @Override
-    public void copyBytes(DataInput input, long numBytes) throws IOException {
-      _indexOutput.copyBytes(input, numBytes);
+    protected void flushBuffer(byte[] b, int offset, int len) throws IOException {
+      _indexOutput.writeBytes(b, offset, len);
       _progressable.progress();
     }
 
     @Override
-    public void flush() throws IOException {
-      _indexOutput.flush();
-      _progressable.progress();
-    }
-
-    @Override
-    public long getFilePointer() {
-      return _indexOutput.getFilePointer();
-    }
-
-    @Override
     public long length() throws IOException {
       return _indexOutput.length();
     }
 
     @Override
-    public void seek(long pos) throws IOException {
-      _indexOutput.seek(pos);
-      _progressable.progress();
-    }
-
-    @Override
-    public void setLength(long length) throws IOException {
-      _indexOutput.setLength(length);
-      _progressable.progress();
-    }
-
-    @Override
-    public String toString() {
-      return _indexOutput.toString();
-    }
-
-    @Override
-    public void writeByte(byte b) throws IOException {
-      _indexOutput.writeByte(b);
-    }
-
-    @Override
-    public void writeBytes(byte[] b, int offset, int length) throws IOException {
-      _indexOutput.writeBytes(b, offset, length);
-      _progressable.progress();
-    }
-
-    @Override
-    public void writeBytes(byte[] b, int length) throws IOException {
-      _indexOutput.writeBytes(b, length);
+    public void close() throws IOException {
+      super.close();
+      _indexOutput.close();
       _progressable.progress();
     }
 
-    @Override
-    public void writeInt(int i) throws IOException {
-      _indexOutput.writeInt(i);
-    }
-
-    @Override
-    public void writeLong(long i) throws IOException {
-      _indexOutput.writeLong(i);
-    }
-
-    @Override
-    public void writeString(String s) throws IOException {
-      _indexOutput.writeString(s);
-    }
-
-    @Override
-    public void writeStringStringMap(Map<String, String> map) throws IOException {
-      _indexOutput.writeStringStringMap(map);
-    }
-
   }
 
   static class ProgressableIndexInput extends BufferedIndexInput {


Mime
View raw message