hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1362666 - in /hadoop/common/branches/branch-1.1: ./ src/core/org/apache/hadoop/io/ src/core/org/apache/hadoop/io/nativeio/ src/mapred/ src/native/ src/native/src/org/apache/hadoop/io/nativeio/ src/test/org/apache/hadoop/io/nativeio/
Date Tue, 17 Jul 2012 21:30:23 GMT
Author: suresh
Date: Tue Jul 17 21:30:23 2012
New Revision: 1362666

URL: http://svn.apache.org/viewvc?rev=1362666&view=rev
Log:
HADOOP-7753. Backporting from branch-1 - Support fadvise and sync_file_range in NativeIO.
Add ReadaheadPool infrastructure for use in HDFS and MR. Contributed by Todd Lipcon and Brandon
Li.

Added:
    hadoop/common/branches/branch-1.1/src/core/org/apache/hadoop/io/ReadaheadPool.java
      - copied unchanged from r1360940, hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/ReadaheadPool.java
Modified:
    hadoop/common/branches/branch-1.1/   (props changed)
    hadoop/common/branches/branch-1.1/CHANGES.txt   (contents, props changed)
    hadoop/common/branches/branch-1.1/src/core/org/apache/hadoop/io/nativeio/NativeIO.java
    hadoop/common/branches/branch-1.1/src/mapred/   (props changed)
    hadoop/common/branches/branch-1.1/src/native/configure.ac
    hadoop/common/branches/branch-1.1/src/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
    hadoop/common/branches/branch-1.1/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c
    hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/io/nativeio/TestNativeIO.java

Propchange: hadoop/common/branches/branch-1.1/
------------------------------------------------------------------------------
  Merged /hadoop/common/branches/branch-1:r1360940

Modified: hadoop/common/branches/branch-1.1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/CHANGES.txt?rev=1362666&r1=1362665&r2=1362666&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1.1/CHANGES.txt Tue Jul 17 21:30:23 2012
@@ -8,6 +8,10 @@ Release 1.1.1 - Unreleased
 
   IMPROVEMENTS
 
+    HADOOP-7753. Support fadvise and sync_file_range in NativeIO. Add
+    ReadaheadPool infrastructure for use in HDFS and MR.
+    (Brandon Li and todd via suresh)
+
   BUG FIXES
 
 

Propchange: hadoop/common/branches/branch-1.1/CHANGES.txt
------------------------------------------------------------------------------
  Merged /hadoop/common/branches/branch-1/CHANGES.txt:r1360940

Modified: hadoop/common/branches/branch-1.1/src/core/org/apache/hadoop/io/nativeio/NativeIO.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/core/org/apache/hadoop/io/nativeio/NativeIO.java?rev=1362666&r1=1362665&r2=1362666&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.1/src/core/org/apache/hadoop/io/nativeio/NativeIO.java
(original)
+++ hadoop/common/branches/branch-1.1/src/core/org/apache/hadoop/io/nativeio/NativeIO.java
Tue Jul 17 21:30:23 2012
@@ -48,10 +48,41 @@ public class NativeIO {
   public static final int O_FSYNC = O_SYNC;
   public static final int O_NDELAY = O_NONBLOCK;
 
+  // Flags for posix_fadvise() from bits/fcntl.h
+  /* No further special treatment. */
+  public static final int POSIX_FADV_NORMAL = 0;
+  /* Expect random page references. */
+  public static final int POSIX_FADV_RANDOM = 1;
+  /* Expect sequential page references. */
+  public static final int POSIX_FADV_SEQUENTIAL = 2;
+  /* Will need these pages. */
+  public static final int POSIX_FADV_WILLNEED = 3;
+  /* Don't need these pages. */
+  public static final int POSIX_FADV_DONTNEED = 4;
+  /* Data will be accessed once. */
+  public static final int POSIX_FADV_NOREUSE = 5;
+
+  /*
+   * Wait upon writeout of all pages in the range before performing the write.
+   */
+  public static final int SYNC_FILE_RANGE_WAIT_BEFORE = 1;
+  /*
+   * Initiate writeout of all those dirty pages in the range which are not
+   * presently under writeback.
+   */
+  public static final int SYNC_FILE_RANGE_WRITE = 2;
+
+  /*
+   * Wait upon writeout of all pages in the range after performing the write.
+   */
+  public static final int SYNC_FILE_RANGE_WAIT_AFTER = 4;
+  
   private static final Log LOG = LogFactory.getLog(NativeIO.class);
 
   private static boolean nativeLoaded = false;
-
+  private static boolean fadvisePossible = true;
+  private static boolean syncFileRangePossible = true;
+  
   static {
     if (NativeCodeLoader.isNativeCodeLoaded()) {
       try {
@@ -94,6 +125,59 @@ public class NativeIO {
       this.username = username;
     }
   }
+  
+  /** Wrapper around posix_fadvise(2) */
+  static native void posix_fadvise(
+    FileDescriptor fd, long offset, long len, int flags) throws NativeIOException;
+
+  /** Wrapper around sync_file_range(2) */
+  static native void sync_file_range(
+    FileDescriptor fd, long offset, long nbytes, int flags) throws NativeIOException;
+
+  /**
+   * Call posix_fadvise on the given file descriptor. See the manpage
+   * for this syscall for more information. On systems where this
+   * call is not available, does nothing.
+   *
+   * @throws NativeIOException if there is an error with the syscall
+   */
+  public static void posixFadviseIfPossible(
+      FileDescriptor fd, long offset, long len, int flags)
+      throws NativeIOException {
+
+    if (nativeLoaded && fadvisePossible) {
+      try {
+        posix_fadvise(fd, offset, len, flags);
+      } catch (UnsupportedOperationException uoe) {
+        fadvisePossible = false;
+      } catch (UnsatisfiedLinkError ule) {
+        fadvisePossible = false;
+      }
+    }
+  }
+
+  /**
+   * Call sync_file_range on the given file descriptor. See the manpage
+   * for this syscall for more information. On systems where this
+   * call is not available, does nothing.
+   *
+   * @throws NativeIOException if there is an error with the syscall
+   */
+  public static void syncFileRangeIfPossible(
+      FileDescriptor fd, long offset, long nbytes, int flags)
+      throws NativeIOException {
+
+    if (nativeLoaded && syncFileRangePossible) {
+      try {
+        sync_file_range(fd, offset, nbytes, flags);
+      } catch (UnsupportedOperationException uoe) {
+        syncFileRangePossible = false;
+      } catch (UnsatisfiedLinkError ule) {
+        syncFileRangePossible = false;
+      }
+    }
+  }
+
   private static final Map<Long, CachedUid> uidCache = 
     new ConcurrentHashMap<Long, CachedUid>();
   private static long cacheTimeout;

Propchange: hadoop/common/branches/branch-1.1/src/mapred/
------------------------------------------------------------------------------
  Merged /hadoop/common/branches/branch-1/src/mapred:r1360940

Modified: hadoop/common/branches/branch-1.1/src/native/configure.ac
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/native/configure.ac?rev=1362666&r1=1362665&r2=1362666&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.1/src/native/configure.ac (original)
+++ hadoop/common/branches/branch-1.1/src/native/configure.ac Tue Jul 17 21:30:23 2012
@@ -39,6 +39,7 @@ AC_CONFIG_SRCDIR([src/org_apache_hadoop.
 AC_CONFIG_AUX_DIR(config)
 AC_CONFIG_HEADER([config.h])
 AC_SYS_LARGEFILE
+AC_GNU_SOURCE
 
 AM_INIT_AUTOMAKE(hadoop,1.0.0)
 
@@ -80,10 +81,8 @@ then
     JNI_CPPFLAGS="$JNI_CPPFLAGS -I$dir"
   done
 fi
-cppflags_bak=$CPPFLAGS
 CPPFLAGS="$CPPFLAGS $JNI_CPPFLAGS"
 AC_CHECK_HEADERS([jni.h], [], AC_MSG_ERROR([Native java headers not found. Is \$JAVA_HOME
set correctly?]))
-CPPFLAGS=$cppflags_bak
 AC_SUBST([JNI_CPPFLAGS])
 
 dnl Check for zlib headers
@@ -95,6 +94,12 @@ AC_CHECK_HEADERS([snappy-c.h], AC_COMPUT
 dnl Check for headers needed by the native Group resolution implementation
 AC_CHECK_HEADERS([fcntl.h stdlib.h string.h unistd.h], [], AC_MSG_ERROR(Some system headers
not found... please ensure their presence on your platform.))
 
+dnl check for posix_fadvise
+AC_CHECK_HEADERS(fcntl.h, [AC_CHECK_FUNCS(posix_fadvise)])
+
+dnl check for sync_file_range
+AC_CHECK_HEADERS(fcntl.h, [AC_CHECK_FUNCS(sync_file_range)])
+
 # Checks for typedefs, structures, and compiler characteristics.
 AC_C_CONST
 

Modified: hadoop/common/branches/branch-1.1/src/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/native/src/org/apache/hadoop/io/nativeio/NativeIO.c?rev=1362666&r1=1362665&r2=1362666&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.1/src/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
(original)
+++ hadoop/common/branches/branch-1.1/src/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
Tue Jul 17 21:30:23 2012
@@ -29,6 +29,7 @@
 #include <string.h>
 #include <sys/stat.h>
 #include <sys/types.h>
+#include <sys/syscall.h>
 #include <unistd.h>
 
 #include "org_apache_hadoop.h"
@@ -170,6 +171,79 @@ cleanup:
   return ret;
 }
 
+/**
+ * public static native void posix_fadvise(
+ *   FileDescriptor fd, long offset, long len, int flags);
+ */
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_posix_1fadvise(
+  JNIEnv *env, jclass clazz,
+  jobject fd_object, jlong offset, jlong len, jint flags)
+{
+#ifndef HAVE_POSIX_FADVISE
+  THROW(env, "java/lang/UnsupportedOperationException",
+        "fadvise support not available");
+#else
+  int fd = fd_get(env, fd_object);
+  PASS_EXCEPTIONS(env);
+
+  int err = 0;
+  if ((err = posix_fadvise(fd, (off_t)offset, (off_t)len, flags))) {
+    throw_ioe(env, err);
+  }
+#endif
+}
+
+#if defined(HAVE_SYNC_FILE_RANGE)
+#  define my_sync_file_range sync_file_range
+#elif defined(SYS_sync_file_range)
+// RHEL 5 kernels have sync_file_range support, but the glibc
+// included does not have the library function. We can
+// still call it directly, and if it's not supported by the
+// kernel, we'd get ENOSYS. See RedHat Bugzilla #518581
+static int manual_sync_file_range (int fd, __off64_t from, __off64_t to, unsigned int flags)
+{
+#ifdef __x86_64__
+  return syscall( SYS_sync_file_range, fd, from, to, flags);
+#else
+  return syscall (SYS_sync_file_range, fd,
+    __LONG_LONG_PAIR ((long) (from >> 32), (long) from),
+    __LONG_LONG_PAIR ((long) (to >> 32), (long) to),
+    flags);
+#endif
+}
+#define my_sync_file_range manual_sync_file_range
+#endif
+
+/**
+ * public static native void sync_file_range(
+ *   FileDescriptor fd, long offset, long len, int flags);
+ */
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_sync_1file_1range(
+  JNIEnv *env, jclass clazz,
+  jobject fd_object, jlong offset, jlong len, jint flags)
+{
+#ifndef my_sync_file_range
+  THROW(env, "java/lang/UnsupportedOperationException",
+        "sync_file_range support not available");
+#else
+  int fd = fd_get(env, fd_object);
+  PASS_EXCEPTIONS(env);
+
+  if (my_sync_file_range(fd, (off_t)offset, (off_t)len, flags)) {
+    if (errno == ENOSYS) {
+      // we know the syscall number, but it's not compiled
+      // into the running kernel
+      THROW(env, "java/lang/UnsupportedOperationException",
+            "sync_file_range kernel support not available");
+      return;
+    } else {
+      throw_ioe(env, errno);
+    }
+  }
+#endif
+}
 
 /*
  * public static native FileDescriptor open(String path, int flags, int mode);

Modified: hadoop/common/branches/branch-1.1/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c?rev=1362666&r1=1362665&r2=1362666&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.1/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c
(original)
+++ hadoop/common/branches/branch-1.1/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c
Tue Jul 17 21:30:23 2012
@@ -54,6 +54,11 @@ void fd_deinit(JNIEnv *env) {
  * underlying fd, or throw if unavailable
  */
 int fd_get(JNIEnv* env, jobject obj) {
+  if (obj == NULL) {
+    THROW(env, "java/lang/NullPointerException",
+          "FileDescriptor object is null");
+    return -1;
+  }
   return (*env)->GetIntField(env, obj, fd_descriptor);
 }
 

Modified: hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/io/nativeio/TestNativeIO.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/io/nativeio/TestNativeIO.java?rev=1362666&r1=1362665&r2=1362666&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/io/nativeio/TestNativeIO.java
(original)
+++ hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/io/nativeio/TestNativeIO.java
Tue Jul 17 21:30:23 2012
@@ -19,6 +19,7 @@ package org.apache.hadoop.io.nativeio;
 
 import java.io.File;
 import java.io.FileDescriptor;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import org.junit.Before;
@@ -163,6 +164,65 @@ public class TestNativeIO {
     assertPermissions(toChmod, 0644);
   }
 
+  @Test
+  public void testPosixFadvise() throws Exception {
+    FileInputStream fis = new FileInputStream("/dev/zero");
+    try {
+      NativeIO.posix_fadvise(fis.getFD(), 0, 0,
+                             NativeIO.POSIX_FADV_SEQUENTIAL);
+    } catch (UnsupportedOperationException uoe) {
+      // we should just skip the unit test on machines where we don't
+      // have fadvise support
+      assumeTrue(false);
+    } finally {
+      fis.close();
+    }
+
+    try {
+      NativeIO.posix_fadvise(fis.getFD(), 0, 1024,
+                             NativeIO.POSIX_FADV_SEQUENTIAL);
+
+      fail("Did not throw on bad file");
+    } catch (NativeIOException nioe) {
+      assertEquals(Errno.EBADF, nioe.getErrno());
+    }
+    
+    try {
+      NativeIO.posix_fadvise(null, 0, 1024,
+                             NativeIO.POSIX_FADV_SEQUENTIAL);
+
+      fail("Did not throw on null file");
+    } catch (NullPointerException npe) {
+      // expected
+    }
+  }
+
+  @Test
+  public void testSyncFileRange() throws Exception {
+    FileOutputStream fos = new FileOutputStream(
+      new File(TEST_DIR, "testSyncFileRange"));
+    try {
+      fos.write("foo".getBytes());
+      NativeIO.sync_file_range(fos.getFD(), 0, 1024,
+                               NativeIO.SYNC_FILE_RANGE_WRITE);
+      // no way to verify that this actually has synced,
+      // but if it doesn't throw, we can assume it worked
+    } catch (UnsupportedOperationException uoe) {
+      // we should just skip the unit test on machines where we don't
+      // have fadvise support
+      assumeTrue(false);
+    } finally {
+      fos.close();
+    }
+    try {
+      NativeIO.sync_file_range(fos.getFD(), 0, 1024,
+                               NativeIO.SYNC_FILE_RANGE_WRITE);
+      fail("Did not throw on bad file");
+    } catch (NativeIOException nioe) {
+      assertEquals(Errno.EBADF, nioe.getErrno());
+    }
+  }
+
   private void assertPermissions(File f, int expected) throws IOException {
     FileSystem localfs = FileSystem.getLocal(new Configuration());
     FsPermission perms = localfs.getFileStatus(



Mime
View raw message