hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject [32/50] hadoop git commit: HDFS-11531. Expose hedged read metrics via libHDFS API. Contributed by Sailesh Mukil.
Date Thu, 20 Apr 2017 21:53:50 GMT
HDFS-11531. Expose hedged read metrics via libHDFS API. Contributed by Sailesh Mukil.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8c81a16a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8c81a16a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8c81a16a

Branch: refs/heads/YARN-5734
Commit: 8c81a16a1fa40d3a3796b1c3633328f1340883ca
Parents: af8e984
Author: John Zhuge <jzhuge@apache.org>
Authored: Tue Apr 18 21:11:07 2017 -0700
Committer: John Zhuge <jzhuge@apache.org>
Committed: Tue Apr 18 21:11:07 2017 -0700

----------------------------------------------------------------------
 .../hadoop/hdfs/DistributedFileSystem.java      |  9 +++
 .../libhdfs-tests/test_libhdfs_threaded.c       | 10 +++
 .../src/main/native/libhdfs/hdfs.c              | 81 ++++++++++++++++++++
 .../src/main/native/libhdfs/include/hdfs/hdfs.h | 29 +++++++
 4 files changed, 129 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c81a16a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 11d7eb8..e1c2105 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -201,6 +201,15 @@ public class DistributedFileSystem extends FileSystem {
   }
 
   /**
+   * Returns the hedged read metrics object for this client.
+   *
+   * @return object of DFSHedgedReadMetrics
+   */
+  public DFSHedgedReadMetrics getHedgedReadMetrics() {
+    return dfs.getHedgedReadMetrics();
+  }
+
+  /**
    * Checks that the passed URI belongs to this filesystem and returns
    * just the path component. Expects a URI with an absolute path.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c81a16a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
index 032acbf..ee1b080 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
@@ -145,6 +145,7 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
     int ret, expected, numEntries;
     hdfsFileInfo *fileInfo;
     struct hdfsReadStatistics *readStats = NULL;
+    struct hdfsHedgedReadMetrics *hedgedMetrics = NULL;
 
     if (hdfsExists(fs, paths->prefix) == 0) {
         EXPECT_ZERO(hdfsDelete(fs, paths->prefix, 1));
@@ -204,6 +205,15 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
     EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalLocalBytesRead);
     EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalShortCircuitBytesRead);
     hdfsFileFreeReadStatistics(readStats);
+
+    /* Verify that we can retrieve the hedged read metrics */
+    EXPECT_ZERO(hdfsGetHedgedReadMetrics(fs, &hedgedMetrics));
+    errno = 0;
+    EXPECT_UINT64_EQ(UINT64_C(0), hedgedMetrics->hedgedReadOps);
+    EXPECT_UINT64_EQ(UINT64_C(0), hedgedMetrics->hedgedReadOpsWin);
+    EXPECT_UINT64_EQ(UINT64_C(0), hedgedMetrics->hedgedReadOpsInCurThread);
+    hdfsFreeHedgedReadMetrics(hedgedMetrics);
+
     /* TODO: implement readFully and use it here */
     ret = hdfsRead(fs, file, tmp, sizeof(tmp));
     if (ret < 0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c81a16a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
index 1dcc768..36c9583 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
@@ -93,6 +93,87 @@ int hdfsFileIsOpenForRead(hdfsFile file)
     return (file->type == HDFS_STREAM_INPUT);
 }
 
+int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics)
+{
+    jthrowable jthr;
+    jobject hedgedReadMetrics = NULL;
+    jvalue jVal;
+    struct hdfsHedgedReadMetrics *m = NULL;
+    int ret;
+    jobject jFS = (jobject)fs;
+    JNIEnv* env = getJNIEnv();
+
+    if (env == NULL) {
+        errno = EINTERNAL;
+        return -1;
+    }
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, jFS,
+                  HADOOP_DFS,
+                  "getHedgedReadMetrics",
+                  "()Lorg/apache/hadoop/hdfs/DFSHedgedReadMetrics;");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "hdfsGetHedgedReadMetrics: getHedgedReadMetrics failed");
+        goto done;
+    }
+    hedgedReadMetrics = jVal.l;
+
+    m = malloc(sizeof(struct hdfsHedgedReadMetrics));
+    if (!m) {
+      ret = ENOMEM;
+      goto done;
+    }
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics,
+                  "org/apache/hadoop/hdfs/DFSHedgedReadMetrics",
+                  "getHedgedReadOps", "()J");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "hdfsGetHedgedReadStatistics: getHedgedReadOps failed");
+        goto done;
+    }
+    m->hedgedReadOps = jVal.j;
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics,
+                  "org/apache/hadoop/hdfs/DFSHedgedReadMetrics",
+                  "getHedgedReadWins", "()J");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "hdfsGetHedgedReadStatistics: getHedgedReadWins failed");
+        goto done;
+    }
+    m->hedgedReadOpsWin = jVal.j;
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics,
+                  "org/apache/hadoop/hdfs/DFSHedgedReadMetrics",
+                  "getHedgedReadOpsInCurThread", "()J");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "hdfsGetHedgedReadStatistics: getHedgedReadOpsInCurThread failed");
+        goto done;
+    }
+    m->hedgedReadOpsInCurThread = jVal.j;
+
+    *metrics = m;
+    m = NULL;
+    ret = 0;
+
+done:
+    destroyLocalReference(env, hedgedReadMetrics);
+    free(m);
+    if (ret) {
+      errno = ret;
+      return -1;
+    }
+    return 0;
+}
+
+void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics)
+{
+  free(metrics);
+}
+
 int hdfsFileGetReadStatistics(hdfsFile file,
                               struct hdfsReadStatistics **stats)
 {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c81a16a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
index 83c1c59..71fa2c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
@@ -169,6 +169,35 @@ extern  "C" {
     LIBHDFS_EXTERNAL
     void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats);
 
+    struct hdfsHedgedReadMetrics {
+      uint64_t hedgedReadOps;
+      uint64_t hedgedReadOpsWin;
+      uint64_t hedgedReadOpsInCurThread;
+    };
+
+    /**
+     * Get cluster wide hedged read metrics.
+     *
+     * @param fs       The configured filesystem handle
+     * @param metrics  (out parameter) on a successful return, the hedged read
+     *                 metrics. Unchanged otherwise. You must free the returned
+     *                 statistics with hdfsFreeHedgedReadMetrics.
+     * @return         0 if the metrics were successfully returned, -1 otherwise.
+     *                 On a failure, please check errno against
+     *                 ENOTSUP. webhdfs, LocalFilesystem, and so forth may
+     *                 not support hedged read metrics.
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics);
+
+    /**
+     * Free HDFS Hedged read metrics.
+     *
+     * @param metrics  The HDFS Hedged read metrics to free
+     */
+    LIBHDFS_EXTERNAL
+    void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics);
+
     /** 
      * hdfsConnectAsUser - Connect to a hdfs file system as a specific user
      * Connect to the hdfs.


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message