asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject [03/34] asterixdb git commit: [NO ISSUE] IOCounter improvements
Date Mon, 09 Apr 2018 21:26:21 GMT
[NO ISSUE] IOCounter improvements

- cache results for short period
- split iostat, proc fs impls

Change-Id: I7789171db6b6d7eea3561c24467af63f065f5dc6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2515
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Tested-by: Murtadha Hubail <mhubail@apache.org>
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/94b6da6a
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/94b6da6a
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/94b6da6a

Branch: refs/heads/release-0.9.4-pre-rc
Commit: 94b6da6ae33749dfdee35e35f32c165693015e3d
Parents: 2587f1d
Author: Michael Blow <mblow@apache.org>
Authored: Thu Mar 22 20:15:37 2018 -0400
Committer: Michael Blow <mblow@apache.org>
Committed: Thu Mar 22 22:11:31 2018 -0700

----------------------------------------------------------------------
 .../control/nc/io/profiling/IIOCounter.java     |   8 +-
 .../control/nc/io/profiling/IOCounterCache.java |  41 ++++++
 .../nc/io/profiling/IOCounterDefault.java       |   6 +-
 .../nc/io/profiling/IOCounterFactory.java       |   9 +-
 .../nc/io/profiling/IOCounterIoStat.java        | 101 +++++++++++++++
 .../control/nc/io/profiling/IOCounterLinux.java | 128 -------------------
 .../control/nc/io/profiling/IOCounterOSX.java   |   6 +-
 .../control/nc/io/profiling/IOCounterProc.java  |  71 ++++++++++
 8 files changed, 231 insertions(+), 139 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java
index 3612d8f..a85ca2c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java
@@ -22,12 +22,12 @@ package org.apache.hyracks.control.nc.io.profiling;
 public interface IIOCounter {
 
     /**
-     * @return the number of block reads from the very beginning
+     * @return the number of block reads from the very beginning, or -1 if not available
on this platform
      */
-    public long getReads();
+    long getReads();
 
     /**
-     * @return the number of block writes from the very beginning
+     * @return the number of block writes from the very beginning, or -1 if not available
on this platform
      */
-    public long getWrites();
+    long getWrites();
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java
new file mode 100644
index 0000000..842d82b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.control.nc.io.profiling;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hyracks.util.Span;
+
+abstract class IOCounterCache<T> implements IIOCounter {
+    private static final long TTL_NANOS = TimeUnit.MILLISECONDS.toNanos(500);
+    private Span span;
+    private T info;
+
+    protected synchronized T getInfo() throws IOException {
+        if (info == null || span.elapsed()) {
+            span = Span.start(TTL_NANOS, TimeUnit.NANOSECONDS);
+            info = calculateInfo();
+        }
+        return info;
+    }
+
+    protected abstract T calculateInfo() throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterDefault.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterDefault.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterDefault.java
index 1f8669d..d5ec9a0 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterDefault.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterDefault.java
@@ -21,14 +21,16 @@ package org.apache.hyracks.control.nc.io.profiling;
 
 public class IOCounterDefault implements IIOCounter {
 
+    public static final long IO_COUNTER_UNAVAILABLE = -1;
+
     @Override
     public long getReads() {
-        return 0;
+        return IO_COUNTER_UNAVAILABLE;
     }
 
     @Override
     public long getWrites() {
-        return 0;
+        return IO_COUNTER_UNAVAILABLE;
     }
 
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java
index 2301ae6..36b310d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java
@@ -33,9 +33,12 @@ public class IOCounterFactory {
      */
     public IIOCounter getIOCounter() {
         String osName = System.getProperty("os.name").toLowerCase();
-        if (osName.indexOf("nix") >= 0 || osName.indexOf("nux") >= 0 || osName.indexOf("aix")
>= 0) {
-            return new IOCounterLinux();
-        } else if (osName.indexOf("mac") >= 0) {
+        if (osName.contains("nix") || osName.contains("nux") || osName.contains("aix")) {
+            if (IOCounterProc.STATFILE.exists()) {
+                return new IOCounterProc();
+            }
+            return new IOCounterIoStat();
+        } else if (osName.contains("mac")) {
             return new IOCounterOSX();
         } else {
             return new IOCounterDefault();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterIoStat.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterIoStat.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterIoStat.java
new file mode 100644
index 0000000..560035b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterIoStat.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.control.nc.io.profiling;
+
+import static org.apache.hyracks.control.nc.io.profiling.IOCounterDefault.IO_COUNTER_UNAVAILABLE;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.StringTokenizer;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class IOCounterIoStat extends IOCounterCache<List<String>> {
+    private static final Logger LOGGER = LogManager.getLogger();
+    private static final String COMMAND = "iostat";
+    private static final int PAGE_SIZE = 512;
+    private long failureCount;
+
+    private long baseReads;
+    private long baseWrites;
+
+    IOCounterIoStat() {
+        baseReads = getReads();
+        baseWrites = getWrites();
+    }
+
+    @Override
+    public long getReads() {
+        try {
+            long reads = extractColumn(4) * PAGE_SIZE;
+            return reads == 0 ? IO_COUNTER_UNAVAILABLE : reads - baseReads;
+        } catch (Exception e) {
+            LOGGER.log(failureCount++ > 0 ? Level.DEBUG : Level.WARN, "Failure getting
reads", e);
+            return IO_COUNTER_UNAVAILABLE;
+        }
+    }
+
+    @Override
+    public long getWrites() {
+        try {
+            long writes = extractColumn(5) * PAGE_SIZE;
+            return writes == 0 ? IO_COUNTER_UNAVAILABLE : writes - baseWrites;
+        } catch (Exception e) {
+            LOGGER.log(failureCount++ > 0 ? Level.DEBUG : Level.WARN, "Failure getting
writes", e);
+            return IO_COUNTER_UNAVAILABLE;
+        }
+    }
+
+    private long extractColumn(int columnIndex) throws IOException {
+        boolean device = false;
+        long ios = 0;
+        for (String line : getInfo()) {
+            if (line.contains("Blk_read")) {
+                device = true;
+                continue;
+            }
+            if (device) {
+                StringTokenizer tokenizer = new StringTokenizer(line);
+                int i = 0;
+                while (tokenizer.hasMoreTokens()) {
+                    String column = tokenizer.nextToken();
+                    if (i == columnIndex) {
+                        ios += Long.parseLong(column);
+                        break;
+                    }
+                    i++;
+                }
+            }
+        }
+        return ios;
+    }
+
+    @Override
+    protected List<String> calculateInfo() throws IOException {
+        try (InputStream inputStream = Runtime.getRuntime().exec(COMMAND).getInputStream())
{
+            return IOUtils.readLines(inputStream, Charset.defaultCharset());
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterLinux.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterLinux.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterLinux.java
deleted file mode 100644
index 3db4c8c..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterLinux.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.control.nc.io.profiling;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.StringTokenizer;
-
-public class IOCounterLinux implements IIOCounter {
-    public static final String COMMAND = "iostat";
-    public static final String STATFILE = "/proc/self/io";
-    public static final int PAGE_SIZE = 4096;
-
-    private long baseReads = 0;
-    private long baseWrites = 0;
-
-    public IOCounterLinux() {
-        baseReads = getReads();
-        baseWrites = getWrites();
-    }
-
-    @Override
-    public long getReads() {
-        try {
-            long reads = extractRow(4);
-            return reads;
-        } catch (IOException e) {
-            try {
-                long reads = extractColumn(4) * PAGE_SIZE;
-                return reads - baseReads;
-            } catch (IOException e2) {
-                return 0;
-            }
-        }
-    }
-
-    @Override
-    public long getWrites() {
-        try {
-            long writes = extractRow(5);
-            long cancelledWrites = extractRow(6);
-            return (writes - cancelledWrites);
-        } catch (IOException e) {
-            try {
-                long writes = extractColumn(5) * PAGE_SIZE;
-                return writes - baseWrites;
-            } catch (IOException e2) {
-                return 0;
-            }
-        }
-    }
-
-    private long extractColumn(int columnIndex) throws IOException {
-        BufferedReader reader = exec(COMMAND);
-        String line = null;
-        boolean device = false;
-        long ios = 0;
-        while ((line = reader.readLine()) != null) {
-            if (line.contains("Blk_read")) {
-                device = true;
-                continue;
-            }
-            if (device == true) {
-                StringTokenizer tokenizer = new StringTokenizer(line);
-                int i = 0;
-                while (tokenizer.hasMoreTokens()) {
-                    String column = tokenizer.nextToken();
-                    if (i == columnIndex) {
-                        ios += Long.parseLong(column);
-                        break;
-                    }
-                    i++;
-                }
-            }
-        }
-        reader.close();
-        return ios;
-    }
-
-    private long extractRow(int rowIndex) throws IOException {
-        BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(STATFILE)));
-        String line = null;
-        long ios = 0;
-        int i = 0;
-        while ((line = reader.readLine()) != null) {
-            if (i == rowIndex) {
-                StringTokenizer tokenizer = new StringTokenizer(line);
-                int j = 0;
-                while (tokenizer.hasMoreTokens()) {
-                    String column = tokenizer.nextToken();
-                    if (j == 1) {
-                        ios = Long.parseLong(column);
-                        break;
-                    }
-                    j++;
-                }
-            }
-            i++;
-        }
-        reader.close();
-        return ios;
-    }
-
-    private BufferedReader exec(String command) throws IOException {
-        Process p = Runtime.getRuntime().exec(command);
-        return new BufferedReader(new InputStreamReader(p.getInputStream()));
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterOSX.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterOSX.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterOSX.java
index 729157b..a48d55d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterOSX.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterOSX.java
@@ -19,16 +19,18 @@
 
 package org.apache.hyracks.control.nc.io.profiling;
 
+import static org.apache.hyracks.control.nc.io.profiling.IOCounterDefault.IO_COUNTER_UNAVAILABLE;
+
 public class IOCounterOSX implements IIOCounter {
 
     @Override
     public long getReads() {
-        return 0;
+        return IO_COUNTER_UNAVAILABLE;
     }
 
     @Override
     public long getWrites() {
-        return 0;
+        return IO_COUNTER_UNAVAILABLE;
     }
 
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterProc.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterProc.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterProc.java
new file mode 100644
index 0000000..8882271
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterProc.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.control.nc.io.profiling;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class IOCounterProc extends IOCounterCache<List<String>> {
+    private static final Logger LOGGER = LogManager.getLogger();
+    @SuppressWarnings("squid:S1075") // hardcoded URI
+    public static final File STATFILE = new File("/proc/self/io");
+    private long failureCount;
+
+    @Override
+    public long getReads() {
+        try {
+            return extractRow(getInfo(), 4);
+        } catch (Exception e) {
+            LOGGER.log(failureCount++ > 0 ? Level.DEBUG : Level.WARN, "Failure getting
reads", e);
+            return IOCounterDefault.IO_COUNTER_UNAVAILABLE;
+        }
+    }
+
+    @Override
+    public long getWrites() {
+        try {
+            List<String> rows = getInfo();
+            long writes = extractRow(rows, 5);
+            long cancelledWrites = extractRow(rows, 6);
+            return writes - cancelledWrites;
+        } catch (Exception e) {
+            LOGGER.log(failureCount++ > 0 ? Level.DEBUG : Level.WARN, "Failure getting
writes", e);
+            return IOCounterDefault.IO_COUNTER_UNAVAILABLE;
+        }
+    }
+
+    private long extractRow(List<String> rows, int rowIndex) {
+        return Long.parseLong(StringUtils.split(rows.get(rowIndex), ' ')[1]);
+    }
+
+    @Override
+    protected List<String> calculateInfo() throws IOException {
+        return FileUtils.readLines(STATFILE, Charset.defaultCharset());
+    }
+
+}


Mime
View raw message