hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject svn commit: r1512878 - in /hadoop/common/branches/branch-1-win: ./ bin/ src/native/ src/native/src/org/apache/hadoop/io/compress/snappy/
Date Sun, 11 Aug 2013 06:27:20 GMT
Author: cnauroth
Date: Sun Aug 11 06:27:20 2013
New Revision: 1512878

URL: http://svn.apache.org/r1512878
Log:
HADOOP-9802. Support Snappy codec on Windows. Contributed by Chris Nauroth.

Modified:
    hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
    hadoop/common/branches/branch-1-win/bin/hadoop-config.cmd
    hadoop/common/branches/branch-1-win/build.xml
    hadoop/common/branches/branch-1-win/src/native/hadoop.vcxproj
    hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
    hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c
    hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h

Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1512878&r1=1512877&r2=1512878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Sun Aug 11 06:27:20 2013
@@ -328,6 +328,8 @@ Branch-hadoop-1-win (branched from branc
     HADOOP-9551. Backport common utils introduced with HADOOP-9413
     to branch-1-win. (ivanmi)
 
+    HADOOP-9802. Support Snappy codec on Windows. (cnauroth)
+
   Merged from branch-1
 
     HDFS-385. Backport: Add support for an experimental API that allows a

Modified: hadoop/common/branches/branch-1-win/bin/hadoop-config.cmd
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/bin/hadoop-config.cmd?rev=1512878&r1=1512877&r2=1512878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/bin/hadoop-config.cmd (original)
+++ hadoop/common/branches/branch-1-win/bin/hadoop-config.cmd Sun Aug 11 06:27:20 2013
@@ -183,7 +183,7 @@ if exist %HADOOP_CORE_HOME%\build\native
 
 @rem For the disto case, check the lib\native folder
 if exist %HADOOP_CORE_HOME%\lib\native (
-  set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;%HADOOP_CORE_HOME%\lib\native\%JAVA_PLATFORM%
+  set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;%HADOOP_CORE_HOME%\lib\native\%JAVA_PLATFORM%;%HADOOP_CORE_HOME%\lib\native
 )
 
 set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.dir=%HADOOP_LOG_DIR%

Modified: hadoop/common/branches/branch-1-win/build.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/build.xml?rev=1512878&r1=1512877&r2=1512878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/build.xml (original)
+++ hadoop/common/branches/branch-1-win/build.xml Sun Aug 11 06:27:20 2013
@@ -702,10 +702,21 @@
 
     <mkdir dir="${build.native}/lib"/>
     <mkdir dir="${build.native.win}/src/org/apache/hadoop/io/compress/zlib"/>
+    <mkdir dir="${build.native.win}/src/org/apache/hadoop/io/compress/snappy"/>
     <mkdir dir="${build.native.win}/src/org/apache/hadoop/io/nativeio"/>
 
     <javah
       classpath="${build.classes}"
+      destdir="${build.native.win}/src/org/apache/hadoop/io/compress/snappy"
+      force="yes"
+      verbose="yes"
+      >
+      <class name="org.apache.hadoop.io.compress.snappy.SnappyCompressor"/>
+      <class name="org.apache.hadoop.io.compress.snappy.SnappyDecompressor"/>
+    </javah>
+
+    <javah
+      classpath="${build.classes}"
       destdir="${build.native.win}/src/org/apache/hadoop/io/compress/zlib"
       force="yes"
       verbose="yes"

Modified: hadoop/common/branches/branch-1-win/src/native/hadoop.vcxproj
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/native/hadoop.vcxproj?rev=1512878&r1=1512877&r2=1512878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/native/hadoop.vcxproj (original)
+++ hadoop/common/branches/branch-1-win/src/native/hadoop.vcxproj Sun Aug 11 06:27:20 2013
@@ -63,7 +63,7 @@
   </ImportGroup>
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup>
-    <IncludePath>$(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;src;..\winutils\include;..\..\build\native\$(OS)-$(Platform)\src\org\apache\hadoop\io\nativeio;..\..\build\native\$(OS)-$(Platform)\src\org\apache\hadoop\io\compress\zlib;$(ZLIB_HOME);$(IncludePath)</IncludePath>
+    <IncludePath>$(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;src;..\winutils\include;..\..\build\native\$(OS)-$(Platform)\src\org\apache\hadoop\io\nativeio;..\..\build\native\$(OS)-$(Platform)\src\org\apache\hadoop\io\compress\zlib;$(ZLIB_HOME);..\..\build\native\$(OS)-$(Platform)\src\org\apache\hadoop\io\compress\snappy;$(SNAPPY_HOME);$(IncludePath)</IncludePath>
   </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
@@ -115,6 +115,8 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="src\org\apache\hadoop\io\compress\snappy\SnappyCompressor.c" Condition="Exists('$(SNAPPY_HOME)')"
/>
+    <ClCompile Include="src\org\apache\hadoop\io\compress\snappy\SnappyDecompressor.c"
Condition="Exists('$(SNAPPY_HOME)')" />
     <ClCompile Include="src\org\apache\hadoop\io\compress\zlib\ZlibCompressor.c" Condition="Exists('$(ZLIB_HOME)')"
/>
     <ClCompile Include="src\org\apache\hadoop\io\compress\zlib\ZlibDecompressor.c" Condition="Exists('$(ZLIB_HOME)')"
/>
     <ClCompile Include="src\org\apache\hadoop\io\nativeio\file_descriptor.c" />
@@ -128,6 +130,7 @@
     <ClInclude Include="..\..\build\native\$(OS)-$(Platform)\src\org\apache\hadoop\io\compress\zlib\org_apache_hadoop_io_compress_zlib_ZlibCompressor.h"
/>
     <ClInclude Include="..\..\build\native\$(OS)-$(Platform)\src\org\apache\hadoop\io\compress\zlib\org_apache_hadoop_io_compress_zlib_ZlibDecompressor.h"
/>
     <ClInclude Include="src\org\apache\hadoop\io\compress\zlib\org_apache_hadoop_io_compress_zlib.h"
/>
+    <ClInclude Include="src\org\apache\hadoop\io\compress\snappy\org_apache_hadoop_io_compress_snappy.h"
/>
     <ClInclude Include="src\org\apache\hadoop\io\nativeio\file_descriptor.h" />
     <ClInclude Include="src\org_apache_hadoop.h" />
   </ItemGroup>

Modified: hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c?rev=1512878&r1=1512877&r2=1512878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
(original)
+++ hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
Sun Aug 11 06:27:20 2013
@@ -20,8 +20,12 @@
   #include <config.h>
 #endif
 
+#include "org_apache_hadoop_io_compress_snappy.h"
+#include "org_apache_hadoop_io_compress_snappy_SnappyCompressor.h"
+
 #if defined HADOOP_SNAPPY_LIBRARY
 
+#ifdef UNIX
 #if defined HAVE_STDIO_H
   #include <stdio.h>
 #else
@@ -45,9 +49,7 @@
 #else
   #error 'dlfcn.h not found'
 #endif
-
-#include "org_apache_hadoop_io_compress_snappy.h"
-#include "org_apache_hadoop_io_compress_snappy_SnappyCompressor.h"
+#endif
 
 static jfieldID SnappyCompressor_clazz;
 static jfieldID SnappyCompressor_uncompressedDirectBuf;
@@ -55,11 +57,18 @@ static jfieldID SnappyCompressor_uncompr
 static jfieldID SnappyCompressor_compressedDirectBuf;
 static jfieldID SnappyCompressor_directBufferSize;
 
+#ifdef UNIX
 static snappy_status (*dlsym_snappy_compress)(const char*, size_t, char*, size_t*);
+#endif
+
+#ifdef WINDOWS
+typedef snappy_status (__cdecl *__dlsym_snappy_compress)(const char*, size_t, char*, size_t*);
+static __dlsym_snappy_compress dlsym_snappy_compress;
+#endif
 
 JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompressor_initIDs
 (JNIEnv *env, jclass clazz){
-
+#ifdef UNIX
   // Load libsnappy.so
   void *libsnappy = dlopen(HADOOP_SNAPPY_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
   if (!libsnappy) {
@@ -68,10 +77,25 @@ JNIEXPORT void JNICALL Java_org_apache_h
     THROW(env, "java/lang/UnsatisfiedLinkError", msg);
     return;
   }
+#endif
+
+#ifdef WINDOWS
+  HMODULE libsnappy = LoadLibrary(HADOOP_SNAPPY_LIBRARY);
+  if (!libsnappy) {
+    THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load snappy.dll");
+    return;
+  }
+#endif
 
   // Locate the requisite symbols from libsnappy.so
+#ifdef UNIX
   dlerror();                                 // Clear any existing error
   LOAD_DYNAMIC_SYMBOL(dlsym_snappy_compress, env, libsnappy, "snappy_compress");
+#endif
+
+#ifdef WINDOWS
+  LOAD_DYNAMIC_SYMBOL(__dlsym_snappy_compress, dlsym_snappy_compress, env, libsnappy, "snappy_compress");
+#endif
 
   SnappyCompressor_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz",
                                                  "Ljava/lang/Class;");
@@ -89,6 +113,9 @@ JNIEXPORT void JNICALL Java_org_apache_h
 
 JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompressor_compressBytesDirect
 (JNIEnv *env, jobject thisj){
+  const char* uncompressed_bytes;
+  char* compressed_bytes;
+  snappy_status ret;
   // Get members of SnappyCompressor
   jobject clazz = (*env)->GetStaticObjectField(env, thisj, SnappyCompressor_clazz);
   jobject uncompressed_direct_buf = (*env)->GetObjectField(env, thisj, SnappyCompressor_uncompressedDirectBuf);
@@ -98,7 +125,7 @@ JNIEXPORT jint JNICALL Java_org_apache_h
 
   // Get the input direct buffer
   LOCK_CLASS(env, clazz, "SnappyCompressor");
-  const char* uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
+  uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
   UNLOCK_CLASS(env, clazz, "SnappyCompressor");
 
   if (uncompressed_bytes == 0) {
@@ -107,14 +134,14 @@ JNIEXPORT jint JNICALL Java_org_apache_h
 
   // Get the output direct buffer
   LOCK_CLASS(env, clazz, "SnappyCompressor");
-  char* compressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
+  compressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
   UNLOCK_CLASS(env, clazz, "SnappyCompressor");
 
   if (compressed_bytes == 0) {
     return (jint)0;
   }
 
-  snappy_status ret = dlsym_snappy_compress(uncompressed_bytes, uncompressed_direct_buf_len,
compressed_bytes, &compressed_direct_buf_len);
+  ret = dlsym_snappy_compress(uncompressed_bytes, uncompressed_direct_buf_len, compressed_bytes,
(size_t* ) &compressed_direct_buf_len);
   if (ret != SNAPPY_OK){
     THROW(env, "java/lang/InternalError", "Could not compress data. Buffer length is too
small.");
   }

Modified: hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c?rev=1512878&r1=1512877&r2=1512878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c
(original)
+++ hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c
Sun Aug 11 06:27:20 2013
@@ -20,8 +20,12 @@
   #include <config.h>
 #endif
 
+#include "org_apache_hadoop_io_compress_snappy.h"
+#include "org_apache_hadoop_io_compress_snappy_SnappyDecompressor.h"
+
 #if defined HADOOP_SNAPPY_LIBRARY
 
+#ifdef UNIX
 #if defined HAVE_STDIO_H
   #include <stdio.h>
 #else
@@ -45,9 +49,7 @@
 #else
   #error 'dlfcn.h not found'
 #endif
-
-#include "org_apache_hadoop_io_compress_snappy.h"
-#include "org_apache_hadoop_io_compress_snappy_SnappyDecompressor.h"
+#endif
 
 static jfieldID SnappyDecompressor_clazz;
 static jfieldID SnappyDecompressor_compressedDirectBuf;
@@ -55,12 +57,20 @@ static jfieldID SnappyDecompressor_compr
 static jfieldID SnappyDecompressor_uncompressedDirectBuf;
 static jfieldID SnappyDecompressor_directBufferSize;
 
+#ifdef UNIX
 static snappy_status (*dlsym_snappy_uncompress)(const char*, size_t, char*, size_t*);
+#endif
+
+#ifdef WINDOWS
+typedef snappy_status (__cdecl *__dlsym_snappy_uncompress)(const char*, size_t, char*, size_t*);
+static __dlsym_snappy_uncompress dlsym_snappy_uncompress;
+#endif
 
 JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyDecompressor_initIDs
 (JNIEnv *env, jclass clazz){
 
   // Load libsnappy.so
+#ifdef UNIX
   void *libsnappy = dlopen(HADOOP_SNAPPY_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
   if (!libsnappy) {
     char* msg = (char*)malloc(1000);
@@ -68,11 +78,27 @@ JNIEXPORT void JNICALL Java_org_apache_h
     THROW(env, "java/lang/UnsatisfiedLinkError", msg);
     return;
   }
+#endif
+
+#ifdef WINDOWS
+  HMODULE libsnappy = LoadLibrary(HADOOP_SNAPPY_LIBRARY);
+  if (!libsnappy) {
+    THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load snappy.dll");
+    return;
+  }
+#endif
 
   // Locate the requisite symbols from libsnappy.so
+#ifdef UNIX
   dlerror();                                 // Clear any existing error
   LOAD_DYNAMIC_SYMBOL(dlsym_snappy_uncompress, env, libsnappy, "snappy_uncompress");
 
+#endif
+
+#ifdef WINDOWS
+  LOAD_DYNAMIC_SYMBOL(__dlsym_snappy_uncompress, dlsym_snappy_uncompress, env, libsnappy,
"snappy_uncompress");
+#endif
+
   SnappyDecompressor_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz",
                                                    "Ljava/lang/Class;");
   SnappyDecompressor_compressedDirectBuf = (*env)->GetFieldID(env,clazz,
@@ -89,6 +115,9 @@ JNIEXPORT void JNICALL Java_org_apache_h
 
 JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyDecompressor_decompressBytesDirect
 (JNIEnv *env, jobject thisj){
+  const char* compressed_bytes = NULL;
+  char* uncompressed_bytes = NULL;
+  snappy_status ret;
   // Get members of SnappyDecompressor
   jobject clazz = (*env)->GetStaticObjectField(env,thisj, SnappyDecompressor_clazz);
   jobject compressed_direct_buf = (*env)->GetObjectField(env,thisj, SnappyDecompressor_compressedDirectBuf);
@@ -98,7 +127,7 @@ JNIEXPORT jint JNICALL Java_org_apache_h
 
   // Get the input direct buffer
   LOCK_CLASS(env, clazz, "SnappyDecompressor");
-  const char* compressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
+  compressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
   UNLOCK_CLASS(env, clazz, "SnappyDecompressor");
 
   if (compressed_bytes == 0) {
@@ -107,14 +136,14 @@ JNIEXPORT jint JNICALL Java_org_apache_h
 
   // Get the output direct buffer
   LOCK_CLASS(env, clazz, "SnappyDecompressor");
-  char* uncompressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
+  uncompressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
   UNLOCK_CLASS(env, clazz, "SnappyDecompressor");
 
   if (uncompressed_bytes == 0) {
     return (jint)0;
   }
 
-  snappy_status ret = dlsym_snappy_uncompress(compressed_bytes, compressed_direct_buf_len,
uncompressed_bytes, &uncompressed_direct_buf_len);
+  ret = dlsym_snappy_uncompress(compressed_bytes, compressed_direct_buf_len, uncompressed_bytes,
&uncompressed_direct_buf_len);
   if (ret == SNAPPY_BUFFER_TOO_SMALL){
     THROW(env, "java/lang/InternalError", "Could not decompress data. Buffer length is too
small.");
   } else if (ret == SNAPPY_INVALID_INPUT){

Modified: hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h?rev=1512878&r1=1512877&r2=1512878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h
(original)
+++ hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h
Sun Aug 11 06:27:20 2013
@@ -20,7 +20,9 @@
 #if !defined ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H
 #define ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H
 
+#include "org_apache_hadoop.h"
 
+#ifdef UNIX
 #if defined HAVE_CONFIG_H
   #include <config.h>
 #endif
@@ -51,8 +53,12 @@
     #error 'jni.h not found'
   #endif
 
-  #include "org_apache_hadoop.h"
-
 #endif //define HADOOP_SNAPPY_LIBRARY
+#endif
+
+#ifdef WINDOWS
+#define HADOOP_SNAPPY_LIBRARY L"snappy.dll"
+#include <snappy-c.h>
+#endif
 
 #endif //ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H



Mime
View raw message