metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject [31/44] metron git commit: METRON-961: Spooky intermittent exceptions in pcap during sync should be handled and explained closes apache/incubator-metron#592
Date Thu, 01 Jun 2017 21:41:50 GMT
METRON-961: Spooky intermittent exceptions in pcap during sync should be handled and explained
closes apache/incubator-metron#592


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fee758b3
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fee758b3
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fee758b3

Branch: refs/heads/Metron_0.4.0
Commit: fee758b3fb2277067421a95054c3e328730decda
Parents: ce381f3
Author: cstella <cestella@gmail.com>
Authored: Sat May 20 09:55:42 2017 -0400
Committer: cstella <cestella@gmail.com>
Committed: Sat May 20 09:55:42 2017 -0400

----------------------------------------------------------------------
 .../metron/spout/pcap/PartitionHDFSWriter.java  | 22 ++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/fee758b3/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java
index 86697db..ebd60dc 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java
@@ -65,11 +65,10 @@ public class PartitionHDFSWriter implements AutoCloseable, Serializable
{
     })
     ,HDFS(new SyncHandler() {
       @Override
-      public void sync(FSDataOutputStream outputStream) throws IOException{
-
+      public void sync(FSDataOutputStream outputStream) throws IOException {
         outputStream.hflush();
         outputStream.hsync();
-        ((HdfsDataOutputStream)outputStream).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
+        ((HdfsDataOutputStream) outputStream).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
       }
     })
     ,LOCAL(new SyncHandler() {
@@ -91,8 +90,13 @@ public class PartitionHDFSWriter implements AutoCloseable, Serializable
{
     }
 
     @Override
-    public void sync(FSDataOutputStream input) throws IOException {
-      func.sync(input);
+    public void sync(FSDataOutputStream input) {
+      try {
+        func.sync(input);
+      }
+      catch(IOException ioe) {
+        LOG.warn("Problems during sync, but this shouldn't be too concerning as long as it's
intermittent: " + ioe.getMessage(), ioe);
+      }
     }
   }
 
@@ -156,7 +160,13 @@ public class PartitionHDFSWriter implements AutoCloseable, Serializable
{
   public void handle(long ts, byte[] value) throws IOException {
     turnoverIfNecessary(ts);
     BytesWritable bw = new BytesWritable(value);
-    writer.append(new LongWritable(ts), bw);
+    try {
+      writer.append(new LongWritable(ts), bw);
+    }
+    catch(ArrayIndexOutOfBoundsException aioobe) {
+      LOG.warn("This appears to be HDFS-7765 (https://issues.apache.org/jira/browse/HDFS-7765),
" +
+              "which is an issue with syncing and not problematic: " + aioobe.getMessage(),
aioobe);
+    }
     numWritten++;
     if(numWritten % config.getSyncEvery() == 0) {
       syncHandler.sync(outputStream);


Mime
View raw message